add packer script to build monitoring server

packer/jambonz-monitoring/README.md

@@ -0,0 +1,84 @@
+# packer-homer
+
+A [packer](https://www.packer.io/) template to build an AMI that runs [homer](https://github.com/sipcapture/homer) VoIP monitoring stack.  The base linux distro is Debian 9 (stretch).  The packer script can build any of the following configurations:
+- an "all in one" AMI that includes homer, postgresql, telegraf, influxdb, and grafana.
+- a homer-only AMU that includes homer, postgresql, telegraf and sends statistics to a remote influxdb/grafana server
+- an influxdb/grafana AMI that includes influxdb and grafana, receiving data from a remote homer server via telegraf
+
+The default settings will build an all-in-one server.
+
+Additionally, the [Node-RED](https://nodered.org) graphical application development environment can be installed as an option to any of the above configuration.  Whilst application is a distinct and seperate function from application monitoring, to reduce the number of EC2 instances it may be desirable in some installations to run Node-RED on the monitoring server.  
+
+> For those not familiar with Node-RED, it is a low-code application develoment tool that can be used to build jambonz applications. 
+
+## Installing 
+
+```
+$  packer build -color=false template.json -var 'homer_password=ShnqaYuivIiyd'
+```
+Builds an all in one AMI
+
+```
+$  packer build -color=false template.json -var 'homer_password=ShnqaYuivIiyd' \
+-var 'install_influxdb=no' -var 'influxdb_ip=55.209.23.30'
+```
+Builds a homer-only AMI that sends data to an influxdb service at http://55.209.23.30:8086
+
+```
+$  packer build -color=false template.json -var 'homer_password=ShnqaYuivIiyd' \
+-var 'install_homer=no'
+```
+Builds an influxdb/grafana AMI
+
+### variables
+There are many variables that can be specified on the `packer build` command line; however defaults (which are shown below) are appropriate for building an "all in one" homer server that includes influxdb and telegraf.
+
+```
+"region": "us-east-1"
+```
+The region to create the AMI in
+
+```
+"ami_description": "homer monitoring"
+```
+AMI description.
+
+```
+"instance_type": "t2.medium"
+```
+EC2 Instance type to use when building the AMI.
+
+```
+"homer_user": "homer_user",
+```
+homer username
+
+```
+"homer_password": "XcapJTqy11LnsYRtxXGPTYQkAnI",
+```
+homer password -- it is strongly recommended that you pass a randomly generated password as a command line variable
+
+```
+"install_nodered":  "yes"
+```
+whether or not to install Node-RED.
+
+```
+"install_influxdb":  "yes"
+```
+whether or not to install influxdb and grafana.
+
+```
+"install_homer":  "yes"
+```
+whether or not to install homer, postgresql and telegraf.
+
+```
+"influxdb_ip" : "127.0.0.1"
+```
+ip address of remote influxdb server.  This should only be specified when building a "homer-only" AMI.
+
+```
+"tag_name": "homer"
+```
+AWS tag Name value

packer/jambonz-monitoring/files/ecosystem.config.js

@@ -0,0 +1,20 @@
+module.exports = {
+  apps : [
+   {
+       name: 'node-red',
+       cwd: '/home/admin/apps/node-red',
+       script: 'packages/node_modules/node-red/red.js',
+       out_file: '/home/admin/.pm2/logs/node-red.log',
+       err_file: '/home/admin/.pm2/logs/node-red.log',
+       combine_logs: true,
+       instance_var: 'INSTANCE_ID',
+       exec_mode: 'fork',
+       instances: 1,
+       autorestart: true,
+       watch: false,
+       max_memory_restart: '1G',
+       env: {
+       },
+   }
+   ]
+};

packer/jambonz-monitoring/files/grafana-dashboard-default.yaml

@@ -0,0 +1,8 @@
+apiVersion: 1
+
+providers:
+  - name: Default
+    type: file
+    folder: 'VoIP'
+    options:
+      path: /var/lib/grafana/dashboards

packer/jambonz-monitoring/files/grafana-datasource.yml

@@ -0,0 +1,34 @@
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: InfluxDB
+    orgId: 1
+  - name: InfluxDB-Telegraf
+    orgId: 1
+
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+- name: InfluxDB
+  type: influxdb
+  access: proxy
+  database: homer
+  user: grafana
+  url: http://127.0.0.1:8086
+  jsonData:
+    timeInterval: "15s"
+  # <bool> allow users to edit datasources from the UI.
+  editable: true
+
+- name: InfluxDB-Telegraf
+  type: influxdb
+  access: proxy
+  database: telegraf
+  user: grafana
+  url: http://127.0.0.1:8086
+  jsonData:
+    timeInterval: "15s"
+  # <bool> allow users to edit datasources from the UI.
+  editable: true

packer/jambonz-monitoring/files/telegraf.conf

@@ -0,0 +1,46 @@
+[[outputs.influxdb]]
+  urls = ["http://influxdb:8086/"] # required
+  database = "telegraf" # required
+  retention_policy = "autogen"
+  write_consistency = "any"
+  timeout = "5s"
+  namedrop = ["hep*"]
+
+[[outputs.influxdb]]
+  urls = ["http://influxdb:8086/"] # required
+  database = "homer" # required
+  retention_policy = ""
+  write_consistency = "any"
+  timeout = "5s"
+  namepass = ["hep*"]
+
+
+[[inputs.socket_listener]]
+   service_address = "udp://127.0.0.1:8094"
+   data_format = "influx"
+
+[[inputs.socket_listener]]
+   service_address = "tcp://127.0.0.1:8094"
+   data_format = "influx"
+
+[[inputs.prometheus]]
+  urls = ["http://127.0.0.1:9096/metrics"]
+  namedrop = ["go*","process*","promhttp*"]
+
+[[inputs.statsd]]
+  protocol = "udp"
+  max_tcp_connections = 250
+  tcp_keep_alive = false
+  # tcp_keep_alive_period = "2h"
+  service_address = ":8125"
+  delete_gauges = true
+  delete_counters = true
+  delete_sets = true
+  delete_timings = true
+  ## Percentiles to calculate for timing & histogram stats.
+  percentiles = [50.0, 90.0, 99.0, 99.9, 99.95, 100.0]
+  metric_separator = "_"
+  datadog_extensions = true
+  allowed_pending_messages = 10000
+  percentile_limit = 1000
+  # read_buffer_size = 65535

packer/jambonz-monitoring/scripts/install_chrony.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+sudo apt-get update
+sudo apt-get install -y chrony
+sudo systemctl enable chrony

packer/jambonz-monitoring/scripts/install_grafana.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+if [ "$1" = "yes" ]; then 
+
+curl -sL https://packages.grafana.com/gpg.key | sudo apt-key add -
+echo "deb https://packages.grafana.com/oss/deb stable main" | sudo tee /etc/apt/sources.list.d/grafana.list
+sudo apt-get update 
+sudo apt-get install -y grafana
+sudo mkdir /var/lib/grafana/dashboards
+sudo mv /tmp/grafana-dashboard-default.yaml /etc/grafana/provisioning/dashboards/default.yaml
+sudo mv /tmp/grafana-datasource.yml /etc/grafana/provisioning/datasources/datasource.yml
+
+sudo mv /tmp/grafana-dashboard-heplify.json /var/lib/grafana/dashboards
+sudo mv /tmp/grafana-dashboard-jambonz.json /var/lib/grafana/dashboards
+
+sudo chmod -R grafana:grafana /var/lib/grafana/dashboards
+sudo chmod -R grafana:grafana /etc/grafana/provisioning/dashboards
+
+sudo systemctl enable grafana-server
+sudo systemctl start grafana-server
+
+fi

packer/jambonz-monitoring/scripts/install_homer.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+if [ "$1" == "yes" ]; then 
+
+DB_USER=$2
+DB_PASS=$3
+
+curl -s https://packagecloud.io/install/repositories/qxip/sipcapture/script.deb.sh | sudo bash
+sudo apt-get install -y homer-app heplify-server
+
+sudo sed -i -e "s/homer_user/$DB_USER/g" /usr/local/homer/etc/webapp_config.json
+sudo sed -i -e "s/homer_password/$DB_PASS/g" /usr/local/homer/etc/webapp_config.json
+sudo homer-app -create-table-db-config 
+sudo homer-app -populate-table-db-config
+sudo sed -i -e "s/DBUser\s*=\s*\"postgres\"/DBUser          = \"$DB_USER\"/g" /etc/heplify-server.toml
+sudo sed -i -e "s/DBPass\s*=\s*\"\"/DBPass          = \"$DB_PASS\"/g" /etc/heplify-server.toml
+sudo sed -i -e "s/PromAddr\s*=\s*\"\"/PromAddr        = \"0.0.0.0:9096\"/g" /etc/heplify-server.toml
+sudo sed -i -e "s/AlegIDs\s*=\s*\[\]/AlegIDs        = \[\"X-CID\"]/g" /etc/heplify-server.toml
+sudo sed -i -e "s/CustomHeader\s*=\s*\[\]/CustomHeader        = \[\"X-Application-Sid\", \"X-Originating-Carrier\", \"X-MS-Teams-Tenant-FQDN\", \"X-Authenticated-User\"]/g" /etc/heplify-server.toml
+
+sudo systemctl enable homer-app
+sudo systemctl restart homer-app
+sudo systemctl status homer-app
+
+sudo systemctl enable heplify-server
+sudo systemctl restart heplify-server
+sudo systemctl status heplify-server
+
+fi

packer/jambonz-monitoring/scripts/install_influxdb.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [ "$1" == "yes" ]; then 
+
+sudo apt-get install -y apt-transport-https
+curl -sL https://repos.influxdata.com/influxdb.key | sudo apt-key add -
+echo "deb https://repos.influxdata.com/debian stretch stable" | sudo tee /etc/apt/sources.list.d/influxdb.list
+sudo apt-get update 
+sudo apt-get install -y influxdb
+sudo systemctl enable influxdb
+sudo systemctl start influxdb
+
+fi

packer/jambonz-monitoring/scripts/install_node_red.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+if [ "$1" == "yes" ]; then 
+
+# install node
+curl -sL https://deb.nodesource.com/setup_10.x | sudo bash - && sudo apt-get install -y nodejs
+sudo npm install -g pino-pretty pm2 pm2-logrotate grunt
+
+#install node-red
+mkdir apps && cd $_
+git clone https://github.com/node-red/node-red.git
+cd node-red
+sudo npm install --unsafe-perm
+grunt build
+
+sudo mv /tmp/ecosystem.config.js /home/admin/apps
+sudo chown -R admin:admin /home/admin/apps
+
+sudo -u admin bash -c "pm2 start /home/admin/apps/ecosystem.config.js"
+sudo env PATH=$PATH:/usr/bin /usr/lib/node_modules/pm2/bin/pm2 startup systemd -u admin --hp /home/admin
+sudo -u admin bash -c "pm2 save"
+sudo systemctl enable pm2-admin.service
+
+fi

packer/jambonz-monitoring/scripts/install_postgresql.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ "$1" == "yes" ]; then
+
+DB_USER=$2
+DB_PASS=$3
+
+wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O- | sudo apt-key add -
+sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main" > /etc/apt/sources.list.d/postgresql.list'
+sudo apt-get update
+sudo apt-get install -y postgresql-12
+sudo systemctl daemon-reload
+sudo systemctl enable postgresql
+sudo systemctl restart postgresql
+
+sudo -u postgres psql -c "CREATE DATABASE homer_config;"
+sudo -u postgres psql -c "CREATE DATABASE homer_data;"
+sudo -u postgres psql -c "CREATE ROLE ${DB_USER} WITH SUPERUSER LOGIN PASSWORD '$DB_PASS';"
+sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE homer_config to ${DB_USER};"
+sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE homer_data to ${DB_USER};"
+
+fi

packer/jambonz-monitoring/scripts/install_telegraf.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ "$1" == "yes" ]; then 
+
+INFLUXDB_IP=$2
+
+curl -sL https://repos.influxdata.com/influxdb.key | sudo apt-key add -
+echo "deb https://repos.influxdata.com/debian stretch stable" | sudo tee /etc/apt/sources.list.d/influxdb.list
+sudo apt-get update 
+sudo apt-get install -y telegraf
+
+sudo cp /tmp/telegraf.conf /etc/telegraf/telegraf.conf
+sudo sed -i -e "s/influxdb:8086/$INFLUXDB_IP:8086/g"  /etc/telegraf/telegraf.conf
+
+sudo systemctl enable telegraf
+sudo systemctl start telegraf
+
+fi

packer/jambonz-monitoring/template.json

@@ -0,0 +1,101 @@
+{
+  "variables": {
+    "region": "us-east-1",
+    "ssh_username": "admin",
+    "ami_description": "jambonz monitoring server", 
+    "instance_type": "t2.medium",
+    "homer_user": "homer_user",
+    "homer_password": "XcapJTqy11LnsYRtxXGPTYQkAnI",
+    "install_influxdb":  "yes",
+    "install_homer":  "yes",
+    "install_nodered":  "yes",
+    "influxdb_ip":  "127.0.0.1",
+    "tag_name": "jambonz-monitoring-server"
+  },
+  "builders": [{
+    "type": "amazon-ebs",
+    "region": "{{user `region`}}",
+    "source_ami_filter": {
+      "filters": {
+        "virtualization-type": "hvm",
+        "name": "debian-stretch-hvm-x86_64-gp2-*",
+        "root-device-type": "ebs"
+      },
+      "owners": ["379101102735"],
+      "most_recent": true
+    },
+    "instance_type": "{{user `instance_type`}}",
+    "ssh_username": "{{user `ssh_username`}}",
+    "ami_name": "jambonz-monitoring-server-all-in-one-{{isotime |clean_resource_name }}",
+    "ami_description": "{{user `ami_description`}}",
+    "launch_block_device_mappings": [
+      {
+        "device_name": "xvda",
+        "volume_size": 40,
+        "volume_type": "gp2",
+        "delete_on_termination": true
+      }
+    ],
+    "security_group_filter": {
+      "filters": {
+        "tag:Class": "packer"
+      }
+    },
+    "tags": {
+      "Name": "{{user `tag_name`}}"
+    },
+    "run_tags": {
+      "Name": "homer-ami-building"
+    }
+  }],
+  "provisioners": [
+    {
+      "type": "shell",
+      "inline": [
+        "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
+        "sudo apt-get update",
+        "sudo apt-get -y install software-properties-common make cmake gcc g++ dirmngr sudo python3-dev \\",
+        "curl wget sed git"
+      ]
+    },
+    {
+      "type": "file",
+      "source": "files/",
+      "destination": "/tmp"
+    },
+    {
+      "type": "shell",
+      "script": "scripts/install_chrony.sh"
+    },
+    {
+      "type": "shell",
+      "execute_command": "chmod +x {{ .Path }}; sudo '{{ .Path }}' {{user `install_nodered`}}",
+      "script": "scripts/install_node_red.sh"
+    },
+    {
+      "type": "shell",
+      "execute_command": "chmod +x {{ .Path }}; sudo '{{ .Path }}' {{user `install_homer`}} {{user `homer_user`}} {{user `homer_password`}}",
+      "script": "scripts/install_postgresql.sh"
+    },
+    {
+      "type": "shell",
+      "execute_command": "chmod +x {{ .Path }}; sudo '{{ .Path }}' {{user `install_homer`}} {{user `influxdb_ip`}}",
+      "script": "scripts/install_telegraf.sh"
+    },
+    {
+      "type": "shell",
+      "execute_command": "chmod +x {{ .Path }}; sudo '{{ .Path }}' {{user `install_homer`}} {{user `homer_user`}} {{user `homer_password`}}",
+      "script": "scripts/install_homer.sh"
+    },
+    {
+      "type": "shell",
+      "execute_command": "chmod +x {{ .Path }}; sudo '{{ .Path }}' {{user `install_influxdb`}}",
+      "script": "scripts/install_influxdb.sh"
+    },
+    {
+      "type": "shell",
+      "execute_command": "chmod +x {{ .Path }}; sudo '{{ .Path }}' {{user `install_influxdb`}}",
+      "script": "scripts/install_grafana.sh"
+    }
+  ]
+}