Alert!

{"type":"doc","content":[{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Latest ","type":"text"},{"text":"Monitoring Chart","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/egovernments/DIGIT-DevOps/tree/DIGIT-2.9LTS-monitoring/deploy-as-code/charts/monitoring"}}]},{"text":" is available here.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"shell"},"content":[{"text":"# Clone git repository\ngit clone https://github.com/egovernments/DIGIT-DevOps.git\ncd DIGIT-DevOps\n\n# Checkout to \"digit-lts-monitoring\" branch\ngit checkout digit-lts-monitoring","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Helmfile","type":"text"}]},{"type":"paragraph","content":[{"text":"Update the environments as required with their relevant file-paths of environment & secrets file and the namespace to be used.","type":"text"}]},{"type":"paragraph","content":[{"text":"In below config ","type":"text"},{"text":"\"demo\"","type":"text","marks":[{"type":"code"}]},{"text":" is the environment with default namespace being set & environment files being provided.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"shell"},"content":[{"text":"# deploy-as-code/helm/charts/monitoring/monitoring-helmfile.yaml\nenvironments:\n demo:\n values:\n - namespace: monitoring\n - ../../environments/egov-demo.yaml\n - ../../environments/egov-demo-secrets.yaml","type":"text"}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Environment Files","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Grafana","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"GitHub OAuth App Creation","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Follow the ","type":"text"},{"text":"GitHub OAuth app","type":"text","marks":[{"type":"link","attrs":{"href":"https://docs.github.com/en/developers/apps/building-oauth-apps/creating-an-oauth-app"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Homepage URL","type":"text"},{"type":"hardBreak"},{"text":"https://","type":"text","marks":[{"type":"code"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Authorization callback URL","type":"text"},{"type":"hardBreak"},{"text":"https:///monitoring/login/github","type":"text","marks":[{"type":"code"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Generate ","type":"text"},{"text":"Client ID","type":"text","marks":[{"type":"code"}]},{"text":" & ","type":"text"},{"text":"Client secret","type":"text","marks":[{"type":"code"}]}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Update ","type":"text"},{"text":"Client ID","type":"text","marks":[{"type":"code"}]},{"text":" & ","type":"text"},{"text":"Client secret","type":"text","marks":[{"type":"code"}]},{"text":" in secrets config.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"yaml"},"content":[{"text":"# deploy-as-code/helm/environments/egov-demo-secrets.yaml\ncluster-configs:\n secrets:\n grafana:\n clientID: \n clientSecret: ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Update environment config to allow GitHub organization & teams specific role-based access","type":"text"}]},{"type":"codeBlock","attrs":{"language":"yaml"},"content":[{"text":"# deploy-as-code/helm/environments/egov-demo.yaml\ngrafana:\n github:\n allowed_organizations: [\"\"]\n role_attribute_path: contains(groups[*], '@/') && 'Viewer'","type":"text"}]}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" Valid roles are ","type":"text"},{"text":"None","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"Viewer","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"Editor","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"Admin","type":"text","marks":[{"type":"code"}]},{"text":" or ","type":"text"},{"text":"GrafanaAdmin","type":"text","marks":[{"type":"code"}]},{"type":"hardBreak"},{"text":"Visit official documentation for more information ","type":"text"},{"text":"Grafana GitHub OAuth","type":"text","marks":[{"type":"link","attrs":{"href":"https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/configure-authentication/github/#configure-role-mapping"}}]}]}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":2},"content":[{"text":"Loki Stack","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Filesystem as a storage","type":"text"}]},{"type":"codeBlock","attrs":{"language":"yaml"},"content":[{"text":"# deploy-as-code/helm/environments/egov-demo.yaml\nloki:\n persistence:\n enabled: true\n accessModes:\n - ReadWriteOnce\n size: 10Gi\n serviceAccount:\n annotations: {}\n additionalConfigs:\n schema_config:\n configs:\n - from: 2020-10-24\n store: boltdb-shipper\n object_store: filesystem ## local filesystem as storage\n schema: v11\n index:\n prefix: index_\n period: 24h\n storage_config:\n boltdb_shipper:\n active_index_directory: /data/loki/index\n cache_location: /data/loki/index_cache\n shared_store: filesystem ## local filesystem as storage\n cache_ttl: 24h\n filesystem:\n directory: /data/loki/chunks\n compactor:\n working_directory: /data/loki/boltdb-shipper-compactor\n shared_store: filesystem ## local filesystem as storage\n retention_enabled: true\n compaction_interval: 168h ## compaction in hours\n table_manager:\n retention_deletes_enabled: true\n retention_period: 168h ## retention in hours","type":"text"}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":4},"content":[{"text":"s3 as storage","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Caution:","type":"text","marks":[{"type":"strong"}]},{"text":" Use the ","type":"text"},{"text":"sub","type":"text","marks":[{"type":"code"}]},{"text":" claim instead of ","type":"text"},{"text":"aud","type":"text","marks":[{"type":"code"}]},{"text":" when setting up Web Identity (OIDC) IAM roles to ensure correct identity matching. ","type":"text"}]}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create AWS Web Identity (OIDC) IAM role with following policy.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"json"},"content":[{"text":"{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"AccessToLokiBucket\",\n \"Effect\": \"Allow\",\n \"Action\": [\n \"s3:PutObject\",\n \"s3:GetObject\",\n \"s3:DeleteObject\",\n \"s3:ListBucket\"\n ],\n \"Resource\": [\n \"arn:aws:s3:::\",\n \"arn:aws:s3:::/*\"\n ]\n }\n ]\n}\n","type":"text"}]},{"type":"paragraph"}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Update s3 details & role ARN in below config.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"yaml"},"content":[{"text":"# deploy-as-code/helm/environments/egov-demo.yaml\nloki:\n persistence:\n enabled: true\n accessModes:\n - ReadWriteOnce\n size: 10Gi\n serviceAccount:\n annotations:\n eks.amazonaws.com/role-arn: ## AWS arn for s3 role \n additionalConfigs:\n schema_config:\n configs:\n - from: 2020-10-24\n store: boltdb-shipper\n object_store: s3 ## AWS s3 as storage\n schema: v11\n index:\n prefix: index_\n period: 24h\n storage_config:\n boltdb_shipper:\n active_index_directory: /data/loki/index\n cache_location: /data/loki/index_cache\n shared_store: s3 ## AWS s3 as storage\n cache_ttl: 24h\n aws:\n s3: s3:/// ## s3 region & bucket\n compactor:\n working_directory: /data/loki/boltdb-shipper-compactor\n shared_store: s3 ## AWS s3 as storage\n retention_enabled: true\n compaction_interval: 168h ## compaction in hours\n table_manager:\n retention_deletes_enabled: true\n retention_period: 168h ## retention in hours\n","type":"text"}]}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" Refer to ","type":"text"},{"text":"official docs","type":"text","marks":[{"type":"link","attrs":{"href":"https://grafana.com/docs/loki/latest/configure/"}}]},{"text":" for detailed configuration","type":"text"}]}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":2},"content":[{"text":"Prometheus","type":"text"}]},{"type":"codeBlock","attrs":{"language":"yaml"},"content":[{"text":"# deploy-as-code/helm/environments/egov-demo.yaml\nprometheus:\n externalLabels:\n cluster: ## provide cluster name \n additionalScrapeConfigs:\n - job_name: 'nginx-ingress-metrics'\n static_configs:\n - targets: [ 'nginx-ingress-controller-metrics.egov:10254' ]\n - job_name: 'blackbox'\n metrics_path: /probe\n params:\n module: [ http_2xx ]\n static_configs:\n - targets:\n - ## add all URLs to monitor\n relabel_configs:\n - source_labels: [ __address__ ]\n target_label: __param_target\n - source_labels: [ __param_target ]\n target_label: instance\n - target_label: __address__\n replacement: prometheus-blackbox-exporter:9115\n - job_name: 'blackbox_exporter'\n static_configs:\n - targets: [ 'prometheus-blackbox-exporter:9115' ]","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Alerting","type":"text"}]},{"type":"codeBlock","attrs":{"language":"yaml"},"content":[{"text":"# deploy-as-code/helm/environments/egov-demo.yaml\nprometheus:\n alertmanager:\n enabled: true","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" Enable Alertmanager present under Prometheus Operator","type":"text"}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Slack Alerts","type":"text"}]},{"type":"codeBlock","attrs":{"language":"yaml"},"content":[{"text":"# deploy-as-code/helm/environments/egov-demo-secrets.yaml\ncluster-configs:\n secrets:\n alertmanager:\n config:\n global:\n slack_api_url: https://hooks.slack.com ## slack webhook URL\n resolve_timeout: 5m\n route:\n group_by: ['alertname']\n group_wait: 30s\n group_interval: 5m\n repeat_interval: 10m\n routes:\n - receiver: slack-notification\n match:\n severity: \"warning|critical\"\n continue: true\n receivers:\n - name: slack-notification\n slack_configs:\n - channel: '' ## slack channel\n send_resolved: true\n username: 'Alertmanager'\n title: |\n [{{ .Status | toUpper }}{{ if eq .Status \"firing\" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }}\n text: |\n {{ range .Alerts -}}\n {{- \"\\n\" -}}\n *Alert:* {{ .Annotations.summary }}\n {{ if .Labels.severity }}*Severity:* `{{ .Labels.severity }}`{{ end }}\n *Cluster:* {{ .Labels.cluster }}\n *Details:*\n {{ .Annotations.description }}\n {{ end }}","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" Generate ","type":"text"},{"text":"Slack Incoming Webhook","type":"text","marks":[{"type":"link","attrs":{"href":"https://api.slack.com/messaging/webhooks"}}]},{"text":" & update ","type":"text"},{"text":"slack_api_url","type":"text","marks":[{"type":"code"}]},{"text":" under global config & ","type":"text"},{"text":"slack-channel","type":"text","marks":[{"type":"code"}]},{"text":" under receivers config.","type":"text"}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Email Alerts","type":"text"}]},{"type":"codeBlock","attrs":{"language":"yaml"},"content":[{"text":"# deploy-as-code/helm/environments/egov-demo-secrets.yaml\ncluster-configs:\n secrets:\n alertmanager:\n config:\n global:\n resolve_timeout: 5m\n route:\n group_by: ['alertname']\n group_wait: 30s\n group_interval: 5m\n repeat_interval: 10m\n routes:\n - receiver: email-notification\n match:\n severity: \"warning|critical\"\n continue: true\n receivers:\n - name: email-notification\n email_configs:\n - to: '' ## reciever's email id\n from: '' ## sender's email id\n smarthost: 'smtp.gmail.com:587' ## \"\" Update SMPT\n auth_username: '' ## configuration\n auth_password: '' ## as per the provider \"\"\n send_resolved: true\n headers:\n subject: |\n [{{ .Status | toUpper }}{{ if eq .Status \"firing\" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.cluster }} - {{ .CommonLabels.alertname }}\n html: |\n \n \n Alert!\n \n \n {{ range .Alerts.Firing }}\n

Alert Name: {{ .CommonLabels.alertname }}
Severity: {{ if eq .Labels.severity \"critical\" }}CRITICAL{{ else if eq .Labels.severity \"warning\" }}WARNING{{ else }}{{ .Labels.severity | toUpper }}{{ end }}
Summary:- {{ .Annotations.summary }}
Cluster:- Cluster
Details:\n
{{ .Annotations.description | replace \"\\n\" \"
\" }}
\n

\n {{ end }}\n ","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" Follow this ","type":"text"},{"text":"article","type":"text","marks":[{"type":"link","attrs":{"href":"https://support.google.com/a/answer/176600?hl=en"}}]},{"text":" in order to setup SMTP server for Gmails","type":"text"}]}]},{"type":"paragraph"},{"type":"rule"},{"type":"paragraph","content":[{"type":"inlineExtension","attrs":{"extensionType":"com.atlassian.confluence.macro.core","extensionKey":"pagetree","parameters":{"macroParams":{"root":{"value":"@parent"},"spaces":{"value":"DD"},"startDepth":{"value":"1"}},"macroMetadata":{"macroId":{"value":"554af4c5cd1754d995cad80fe9e51db4ddf93f36e5606c90c5243944470e3b8c"},"schemaVersion":{"value":"1"},"title":"Page Tree"}},"localId":"6edfeb4a-eb7e-4372-8674-eec44ad84578"}}]},{"type":"paragraph"}],"version":1}