From 0d8b28a14cbeedf8f46d83dbe1064bca7d2d9e28 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 21 Apr 2022 00:12:25 -0400 Subject: [PATCH 01/31] Set Prometheus scrape interval to 15s I think the default scrape interval changed from 15s to 1m, but Grafana defaults to 15s and I guess doesn't query Prometheus for it. This breaks graphs such as CPU usage or network transfer, because they don't have enough data to do a `rate`. --- files/prometheus.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/files/prometheus.yml b/files/prometheus.yml index 25acecb..0d4c991 100644 --- a/files/prometheus.yml +++ b/files/prometheus.yml @@ -1,5 +1,8 @@ --- +global: + scrape_interval: 15s + scrape_configs: - job_name: prometheus static_configs: From 8950e97807a8c538b451105536603e559de8c650 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 21 Apr 2022 01:01:54 -0400 Subject: [PATCH 02/31] Only reload Prometheus on config change Since it supports hot-reload of config, there's no need for a full restart. --- matplotlib.org.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/matplotlib.org.yml b/matplotlib.org.yml index 2f07d86..2b5f2c4 100644 --- a/matplotlib.org.yml +++ b/matplotlib.org.yml @@ -246,7 +246,7 @@ dest: /etc/prometheus/prometheus.yml mode: 0644 notify: - - Restart Prometheus + - Reload Prometheus - name: Enable prometheus node exporter service ansible.builtin.systemd: @@ -322,6 +322,11 @@ name: caddy state: reloaded + - name: Reload Prometheus + ansible.builtin.systemd: + name: prometheus + state: reloaded + - name: Restart Prometheus ansible.builtin.systemd: name: prometheus From 65dff8e581d1613db150363d0a053d85df8bfd37 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 21 Apr 2022 01:08:00 -0400 Subject: [PATCH 03/31] Refresh node statistics dashboard This regenerates the dashboard using the "Export for sharing externally" option, which removes some hard-coded information. --- files/grafana/node-statistics.json | 344 +++++++++++++++++------------ 1 file changed, 201 insertions(+), 143 deletions(-) diff --git a/files/grafana/node-statistics.json b/files/grafana/node-statistics.json index 250c97f..28da247 100644 --- a/files/grafana/node-statistics.json +++ b/files/grafana/node-statistics.json @@ -1,4 +1,46 @@ { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.5.11" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], "annotations": { "list": [ { @@ -16,8 +58,8 @@ "editable": true, "gnetId": 1860, "graphTooltip": 0, - "id": 9, - "iteration": 1647041923033, + "id": null, + "iteration": 1650501486555, "links": [ { "icon": "external link", @@ -37,7 +79,7 @@ "panels": [ { "collapsed": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -52,7 +94,7 @@ }, { "cacheTimeout": null, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "Busy state of all CPU cores together", "fieldConfig": { "defaults": { @@ -128,7 +170,7 @@ }, { "cacheTimeout": null, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "Busy state of all CPU cores together (5 min average)", "fieldConfig": { "defaults": { @@ -204,7 +246,7 @@ }, { "cacheTimeout": null, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "Busy state of all CPU cores together (15 min average)", "fieldConfig": { "defaults": { @@ -279,7 +321,7 @@ }, { "cacheTimeout": null, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "Non available RAM memory", "fieldConfig": { "defaults": { @@ -357,7 +399,7 @@ }, { "cacheTimeout": null, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "Used Swap", "fieldConfig": { "defaults": { @@ -431,7 +473,7 @@ }, { "cacheTimeout": null, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "Used Root FS", "fieldConfig": { "defaults": { @@ -513,7 +555,7 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "Total number of CPU cores", "fieldConfig": { "defaults": {}, @@ -601,7 +643,7 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 1, "description": "System uptime", "fieldConfig": { @@ -691,7 +733,7 @@ "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 0, "description": "Total RootFS", "fieldConfig": { @@ -780,7 +822,7 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 0, "description": "Total RAM", "fieldConfig": { @@ -867,7 +909,7 @@ "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 0, "description": "Total SWAP", "fieldConfig": { @@ -947,7 +989,7 @@ }, { "collapsed": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -980,7 +1022,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "description": "Basic CPU info", "fieldConfig": { @@ -1171,7 +1213,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "description": "Basic memory usage", "fieldConfig": { @@ -1354,7 +1396,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "Basic network info per interface", "fieldConfig": { "defaults": { @@ -1472,7 +1514,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 3, "description": "Disk space used of all filesystems mounted", "fieldConfig": { @@ -1575,7 +1617,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -1600,7 +1642,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "description": "", "fieldConfig": { @@ -1792,7 +1834,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "description": "", "fieldConfig": { @@ -1980,7 +2022,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -2096,7 +2138,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 3, "description": "", "fieldConfig": { @@ -2202,7 +2244,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -2402,7 +2444,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 3, "description": "", "fieldConfig": { @@ -2546,7 +2588,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 3, "description": "", "fieldConfig": { @@ -2657,7 +2699,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -2689,7 +2731,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -2818,7 +2860,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -2958,7 +3000,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -3109,7 +3151,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -3246,7 +3288,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -3408,7 +3450,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -3538,7 +3580,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -3682,7 +3724,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -3804,7 +3846,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -3938,7 +3980,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -4069,7 +4111,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -4207,7 +4249,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -4337,7 +4379,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -4484,7 +4526,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -4615,7 +4657,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -4724,7 +4766,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -4738,7 +4780,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -4852,7 +4894,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -4984,7 +5026,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -5132,7 +5174,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 2, "fieldConfig": { "defaults": { @@ -5240,7 +5282,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -5254,7 +5296,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -5384,7 +5426,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -5485,7 +5527,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -5602,7 +5644,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -5713,7 +5755,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -5727,7 +5769,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -5837,7 +5879,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -5940,7 +5982,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -6043,7 +6085,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -6181,7 +6223,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -6301,7 +6343,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -6419,7 +6461,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -6540,7 +6582,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -6554,7 +6596,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -6664,7 +6706,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -6783,7 +6825,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -6894,7 +6936,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -6997,7 +7039,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -7099,7 +7141,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -7202,7 +7244,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -7320,7 +7362,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -7334,7 +7376,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -7488,7 +7530,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -7607,7 +7649,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -7711,7 +7753,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -7725,7 +7767,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -7825,7 +7867,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -7983,7 +8025,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -7997,7 +8039,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "The number (after merges) of I/O requests completed per second for the device", "fieldConfig": { "defaults": { @@ -8215,7 +8257,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "The number of bytes read from or written to the device per second", "fieldConfig": { "defaults": { @@ -8413,7 +8455,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", "fieldConfig": { "defaults": { @@ -8615,7 +8657,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "The average queue length of the requests that were issued to the device", "fieldConfig": { "defaults": { @@ -8803,7 +8845,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "The number of read and write requests merged per second that were queued to the device", "fieldConfig": { "defaults": { @@ -9001,7 +9043,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially. But for devices serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.", "fieldConfig": { "defaults": { @@ -9197,7 +9239,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", "fieldConfig": { "defaults": { @@ -9385,7 +9427,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -9601,7 +9643,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -9615,7 +9657,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": 3, "description": "", "fieldConfig": { @@ -9740,7 +9782,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -9845,7 +9887,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -9954,7 +9996,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -10059,7 +10101,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": null, "description": "", "fieldConfig": { @@ -10177,7 +10219,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -10196,7 +10238,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -10312,7 +10354,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -10430,7 +10472,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -10548,7 +10590,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -10666,7 +10708,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -10776,7 +10818,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -10894,7 +10936,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -11008,7 +11050,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -11113,7 +11155,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -11227,7 +11269,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -11344,7 +11386,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -11444,7 +11486,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -11545,7 +11587,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -11646,7 +11688,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -11747,7 +11789,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -11870,7 +11912,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -11978,7 +12020,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -12092,7 +12134,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -12106,7 +12148,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -12253,7 +12295,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -12380,7 +12422,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -12495,7 +12537,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -12617,7 +12659,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -12725,7 +12767,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -12739,7 +12781,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -12863,7 +12905,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -12972,7 +13014,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": null, "fieldConfig": { "defaults": { @@ -13093,7 +13135,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": null, "fieldConfig": { "defaults": { @@ -13205,7 +13247,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": null, "fieldConfig": { "defaults": { @@ -13330,7 +13372,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -13472,7 +13514,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "decimals": null, "fieldConfig": { "defaults": { @@ -13598,7 +13640,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -13744,7 +13786,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -13869,7 +13911,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -14003,7 +14045,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "fieldConfig": { "defaults": { "links": [] @@ -14119,7 +14161,7 @@ }, { "collapsed": true, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "gridPos": { "h": 1, "w": 24, @@ -14133,7 +14175,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -14237,7 +14279,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": "${DS_PROMETHEUS}", "description": "", "fieldConfig": { "defaults": { @@ -14389,12 +14431,8 @@ }, { "allValue": null, - "current": { - "selected": false, - "text": "node", - "value": "node" - }, - "datasource": "Prometheus", + "current": {}, + "datasource": "${DS_PROMETHEUS}", "definition": "", "description": null, "error": null, @@ -14420,12 +14458,8 @@ }, { "allValue": null, - "current": { - "selected": false, - "text": "localhost:9100", - "value": "localhost:9100" - }, - "datasource": "Prometheus", + "current": {}, + "datasource": "${DS_PROMETHEUS}", "definition": "label_values(node_uname_info{job=\"$job\"}, instance)", "description": null, "error": null, @@ -14480,7 +14514,31 @@ "from": "now-24h", "to": "now" }, - "timepicker": {}, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, "timezone": "browser", "title": "Node Statistics", "uid": "rYdddlPWk", From 94b83af896ece8063e37b5c64ef2d71b5e340a59 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 21 Apr 2022 21:44:16 -0400 Subject: [PATCH 04/31] Enable automatic updates Theoretically, this might break things, but Fedora is quite stable, and I don't think this server is critical enough that being secure is less important than never changing. --- matplotlib.org.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/matplotlib.org.yml b/matplotlib.org.yml index 2f07d86..20c98cd 100644 --- a/matplotlib.org.yml +++ b/matplotlib.org.yml @@ -41,7 +41,9 @@ - name: Install server maintenance ansible.builtin.dnf: - name: "fail2ban" + name: + - dnf-automatic + - fail2ban state: present - name: Install web server requirements @@ -64,6 +66,14 @@ - podman state: present + # Automatic updates + # ################# + - name: Enable automatic updates + ansible.builtin.systemd: + name: dnf-automatic-install.timer + enabled: true + state: started + # Firewall setup # ############## - name: Setup firewall From 00fc1f30f6c6c82bc1bddde901c34f615352f277 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Tue, 3 May 2022 15:31:12 -0400 Subject: [PATCH 05/31] Configure TLS certs before Caddy Otherwise, the validation step fails on a completely new system since the files are missing. --- matplotlib.org.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/matplotlib.org.yml b/matplotlib.org.yml index 2f07d86..e07af36 100644 --- a/matplotlib.org.yml +++ b/matplotlib.org.yml @@ -161,13 +161,6 @@ - name: Caddy setup tags: caddy block: - - name: Configure Caddy - ansible.builtin.template: - src: Caddyfile.j2 - dest: /etc/caddy/Caddyfile - validate: "caddy validate --adapter caddyfile --config %s" - notify: Reload Caddy - - name: Configure Caddy TLS certificate directory ansible.builtin.file: path: /etc/caddy/tls @@ -194,6 +187,13 @@ notify: - Reload Caddy + - name: Configure Caddy + ansible.builtin.template: + src: Caddyfile.j2 + dest: /etc/caddy/Caddyfile + validate: "caddy validate --adapter caddyfile --config %s" + notify: Reload Caddy + - name: Enable Caddy service ansible.builtin.systemd: name: caddy.service From d1e9329c68646aafd90d9d933673bbf7120cc9a9 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Tue, 3 May 2022 18:41:52 -0400 Subject: [PATCH 06/31] Bump droplet size to $10 --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b0ae04b..3c3b757 100644 --- a/README.md +++ b/README.md @@ -114,16 +114,16 @@ The summary of the initial setup is: 3. Grab the SSH host fingerprints. 4. Reboot. -We currently use a simple $5 droplet from DigitalOcean. You can create one from -the control panel, or using the `doctl` utility. Be sure to enable monitoring, -and add the `website` tag and relevant SSH keys to the droplet. An example of -using `doctl` is the following: +We currently use a simple $10 droplet from DigitalOcean. You can create one +from the control panel, or using the `doctl` utility. Be sure to enable +monitoring, and add the `website` tag and relevant SSH keys to the droplet. An +example of using `doctl` is the following: ``` doctl compute droplet create \ --image fedora-35-x64 \ --region tor1 \ - --size s-1vcpu-1gb \ + --size s-1vcpu-2gb \ --ssh-keys , \ --tag-name website \ --enable-monitoring \ @@ -136,7 +136,7 @@ of the new droplet from the output, e.g., in: ``` ID Name Public IPv4 Private IPv4 Public IPv6 Memory VCPUs Disk Region Image VPC UUID Status Tags Features Volumes -294098687 mpl.org 1024 1 25 tor1 Fedora 35 x64 new website monitoring,droplet_agent +294098687 mpl.org 2048 1 50 tor1 Fedora 35 x64 new website monitoring,droplet_agent ``` the droplet ID is 294098687. From aefafea09018e8fdec5457a2e470d8040d39797a Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Wed, 22 Jun 2022 15:46:16 -0400 Subject: [PATCH 07/31] Hide .git folders from website This is generally considered a security issue, but it's kind of meaningless for us since we are serving public repos that contain no secrets, nor are running any webapps that could accidentally expose anything from it. But we can hide to satisfy security scanners. --- templates/Caddyfile.j2 | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index 317982a..5e721fd 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -8,7 +8,9 @@ handle_path /{args.0}/* { root * {{ caddy.site_dir }}/{args.0} try_files {path}.html {path} - file_server + file_server { + hide .git + } } } @@ -77,7 +79,9 @@ http://{{ caddy.addresses.main }}, http://{{ ansible_fqdn }} { handle { root * {{ caddy.site_dir }}/matplotlib.github.com try_files {path}.html {path} - file_server + file_server { + hide .git + } } # Use a custom 404 error page. @@ -85,6 +89,8 @@ http://{{ caddy.addresses.main }}, http://{{ ansible_fqdn }} { @notfound expression {http.error.status_code} == 404 root * {{ caddy.site_dir }}/mpl-brochure-site rewrite @notfound /404.html - file_server + file_server { + hide .git + } } } From bdeaafbcecca7351c22644909a6d0022f4e6068e Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 18 Aug 2022 01:56:38 -0400 Subject: [PATCH 08/31] Restart webhook service on failure --- webhook/webhook.service | 1 + 1 file changed, 1 insertion(+) diff --git a/webhook/webhook.service b/webhook/webhook.service index 031fdcf..f99d996 100644 --- a/webhook/webhook.service +++ b/webhook/webhook.service @@ -11,6 +11,7 @@ User=caddy Group=caddy ExecStart=/usr/bin/python3 /usr/bin/webhook.py localhost:1234 EnvironmentFile=/etc/caddy/webhook.env +Restart=on-failure TimeoutStopSec=5s LimitNOFILE=1048576 LimitNPROC=512 From ddd0b50b7e64e6bb5849af0c2b319ab9b80060e8 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 18 Aug 2022 03:12:50 -0400 Subject: [PATCH 09/31] Add a ping endpoint to webhook service As noted in #25, the webhook was down for some period. The ping endpoint allows some external check to verify that it is running. If Caddy cannot connect to the webhook, it will return 502 instead of 200. --- templates/Caddyfile.j2 | 10 ++++++++++ webhook/test_webhook.py | 9 +++++++++ webhook/webhook.py | 6 ++++++ 3 files changed, 25 insertions(+) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index 5e721fd..d8b4b61 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -46,6 +46,16 @@ http://{{ caddy.addresses.webhook }} { } } + handle /ping { + reverse_proxy * localhost:1234 { + # Don't leak out internal problems. + @error status 4xx 5xx + handle_response @error { + error 503 + } + } + } + handle { error 404 } diff --git a/webhook/test_webhook.py b/webhook/test_webhook.py index dbe0e13..8668ab7 100644 --- a/webhook/test_webhook.py +++ b/webhook/test_webhook.py @@ -65,6 +65,15 @@ async def test_update_repo(tmp_path_factory): assert dest_commit == src_commit +async def test_ping(aiohttp_client, monkeypatch, tmp_path): + """Test ping always works.""" + monkeypatch.setenv('SITE_DIR', str(tmp_path)) + client = await aiohttp_client(create_app()) + + resp = await client.get('/ping') + assert resp.status == 200 + + async def test_github_webhook_errors(aiohttp_client, monkeypatch, tmp_path): """Test invalid inputs to webhook.""" monkeypatch.setenv('SITE_DIR', str(tmp_path)) diff --git a/webhook/webhook.py b/webhook/webhook.py index 7d4e4f2..5d86d17 100644 --- a/webhook/webhook.py +++ b/webhook/webhook.py @@ -173,6 +173,11 @@ async def github_webhook(request: web.Request): return web.Response(status=200) +async def ping(request: web.Request): + """Respond to a ping, thus verifying the webhook service is alive.""" + return web.Response(status=200) + + def create_app(): """Create the aiohttp app and setup routes.""" site_dir = Path(os.environ.get('SITE_DIR', 'sites')).resolve() @@ -182,6 +187,7 @@ def create_app(): app['site_dir'] = site_dir app.add_routes([ web.post('/gh/{repo}', github_webhook), + web.get('/ping', ping), ]) return app From 414a464c0c1df0848c09f04beb015abd3cfb1399 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 26 Aug 2022 15:41:56 -0400 Subject: [PATCH 10/31] Add mpl-sphinx-theme to website --- README.md | 1 + matplotlib.org.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 3c3b757..d83ffe4 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ related projects are also exposed as toplevel directories: * [mpl-altair](https://github.com/matplotlib/mpl-altair) * [mpl-bench](https://github.com/matplotlib/mpl-bench) * [mpl-gui](https://github.com/matplotlib/mpl-gui) +* [mpl-sphinx-theme](https://github.com/matplotlib/mpl-sphinx-theme) * [mpl-third-party](https://github.com/matplotlib/mpl-third-party) Ansible configuration diff --git a/matplotlib.org.yml b/matplotlib.org.yml index 5fe6414..86b10d5 100644 --- a/matplotlib.org.yml +++ b/matplotlib.org.yml @@ -20,6 +20,7 @@ - mpl-altair - mpl-bench - mpl-gui + - mpl-sphinx-theme - mpl-third-party - pytest-mpl vars_files: From 6c68fec47706261c942a05ce09fad066ab9ab29d Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 26 Aug 2022 17:41:40 -0400 Subject: [PATCH 11/31] Add a webhook secret for mpl-sphinx-theme --- files/webhook_vars.yml | 91 ++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/files/webhook_vars.yml b/files/webhook_vars.yml index 9c972f9..19f1b83 100644 --- a/files/webhook_vars.yml +++ b/files/webhook_vars.yml @@ -1,45 +1,48 @@ $ANSIBLE_VAULT;1.1;AES256 -64366462653335313637376437666537643035346339383262656661396131643839656366633665 -3432323262323731393465393339636436623236663761330a373361646261663163613436366436 -62633234633735613037623461303034643136313434633635326464383431323139306431613136 -3930356337623035340a383934323139383839333066646531613862363131623437323261373561 -39383334336334616265363131613762326263353439323563343734653961393438303139653430 -64373361643834643263613037666234656466633836363961663966616439356639353331323166 -32306264373935346364633637326266386433303331343432373239323230333139646466306435 -32336531373230643664306233316564363861383735373536376337363433383532396239313438 -34383266666263313631616636626465633063316564616131386138356430663037663935633131 -61383936373764303564313065646462636563333738333436393832386164653066396434623266 -34336536386233373932623766363832353461303534346562613633323430306530613138356133 -33623131663932623464303336633362356562396464626466366339386136303563613630346132 -64653932643233336464653334306537363964633335626538623033356362386361646634396234 -65653461373364303035323536343536643330363539376666323337376335336566316436646563 -38376133356538643034396231623634393738656333666465616434396666366632313065343534 -34653836623238313238333339303461623337373238646530353539656432306463323336663131 -38666433633034373833323864626366393530346639663837623063316636373532316333663032 -65383837626461306337323233323931356638616233393831323732643164656266343334656665 -35623661366363663164623031633734626362303536613137373162353266633137383937316330 -37303566643264356435653537313566363762353732373266373439613266323962643961333337 -38666464653332306537336333313739396438626338396163623037656437316261613937626633 -65613039633035386530383437303630663232633339363264663333356434306164343866303738 -33623237613635613764313565623564646661633437626636396631356661316231313464373530 -62316166313632386235316332623431383033383164363364343137313839366234373736323333 -30376238366464653430366636366136353765653036316436396636323261373931373332346230 -33316663656132626133336464623132353538633531396538323466356235396633323333653061 -63313666653535663565633835636235303338316363646362663165616337323139323861316636 -64643434616363333366633031633538326266646536326232353064366636623135633337303264 -61366665343330313835623733666338333265313837336139383331613364353936356163316132 -34303835663262303332343931333966353165333138343163386430666538393834363034666337 -33363731323837376631333939326665336464376264366663313330613338666663373737626136 -35653632336466643239616364346430636163303934636633383264636239343131333166643030 -64393662646166626561626531353435323537316165333435623038373132623263363739336633 -62383561623132663135353061663763623339646639656534383464626666633664636437343166 -62333338616336383936373562646336346334383032373230663136323466393737393439613266 -31613339656166656162623232366335383766313166633262303637393165386462353666336439 -30366434386232666364636461343465623862613832646634383934636466653364316436383666 -61663639303238313561373230353734373734353830636161356332616536616234326265633462 -35343164636466353335363232616539363638663834323739396365333462633233663637656335 -66646665396265326639353134303834326630343537303135333538643635633437373463666565 -35393961613864643033623337333662306563653163343662643835383232346233303664653639 -63353331353639663833333335363639633766636130313035353262353364363564343666336230 -39346565633535316363666633646532663931376231383930636438363438653565366634626362 -30623134666362393732 +62303938313862636161613161643430303838323031396332346336373531666361313731363737 +3365333464376237326365346336653561626435366530310a373936386439656533623238623638 +66393435653835363535646438336234356331306163646235316239633538366639353135376635 +6234326362333662630a303830663565623061363561636330333363393130653266306339303631 +62363862623934306539323765636466336238643664626237326362366434353166663963383534 +35356663373533363863376534386532393732623066393832366436323630393265323436393064 +64303439323363306665323761306262623663636661333134643933366262306136643037313262 +38613437343266353464663964343666653366356563323934353164376266626463353864353336 +63323065626166373062666632616332643338323638303261633363633935316637653434356632 +63653964386262386363616662353634643836616565613465613033366237343031616430663766 +63666439343638323262343936373137653239626438626565333336383365343832626661383663 +39356163346638323630343262646261333862313730623431663164303435393837626231326537 +34313733346261623530663762373731643332396664383061643232393638346631373734393230 +31396565633531636439626562643466303930646562323734613139613865653266313837366663 +35353365613764623530313430363465643164643435636464633165313066653961363432613864 +31353665613830333866356132633464633937313231376230653631633133376161653861343463 +65613630633236613531373262633936373934343735383431306337376562323933353963633739 +31313736326238313232626532353331333230656665373633663863653065386536306435363237 +31313936366130303134333036663764303665383735663130303963653832303538633339316561 +39326634633166613934333937663430626433613961353831343934663538323835373365633166 +31366135333536323734343936653162636562376234616635363136643530636166653337313435 +31343632646564616666653162353064366336633239633034636563623336313034313337316335 +38316566343938623265336662643234653964373034306635636135353339306535663962356130 +62616663396339333531313061323732376365663565616663313465653062303866383235353832 +36366531643064303731353230323634376538306139346364633662626261386638343632383030 +30333364363530373131393765396533303431643939636361643231313437666334323532346162 +37353538623233643738366135376339336635623132653337376563353334383362363066343863 +39663462316330663062336232663638623062333432643566386233333963373938623864666439 +61643637333638633734353464333733343735303737323864663733316265333865623864373963 +36386262633036393734353439623434306139303331353464633637663261646438306137613638 +31336463363738643665383461333432643066383961663638353133626236393738626637363533 +34656666346630346631663833333837656130666333666230653136363634386233353262376537 +36333233306561666133346537396639306332336136363738626635363466346132326463356163 +32366166663336343262313933333162353630623064623338633265313464393636623936633033 +61646237366361303164663663313139386432343533666666393131353631626330383961666134 +66653630306332633438323336306363333737646334356465646136613235353362626161633064 +32333931376530666263363863333661333938653237353934303439336334353630353262636435 +39373536396465666165643261303237636364303138616239636438353864386634346138303738 +34626130356262633833373231353066653837303963303930623931363435313537383539366432 +34376661653963343964346663336533346565373435366663363831636338363230373862366235 +35323431653830656634376636373033356337626533353766386534363465343634613161396366 +38386131616534326135646134353832393765393834656239663464306530613937346266396232 +31626561626530613761323665363833653037633761313438383139303565653131386264613963 +31666166383738633938663536643864303866656666333439386133323634646537653865613138 +64373339373338383065666235306665663238653666393361313837333832373334626433663633 +36373765346165613033643734626364663334633433343865616539623564353630313663636338 +663137626238383764616238333262343630 From d0e511160a59be83cd19880328c7d724c6f64f30 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Tue, 30 Aug 2022 05:06:36 -0400 Subject: [PATCH 12/31] Use consistent branch for matplotlib.github.com The webhook uses `main`, but the playbook always used `gh-pages`. This caused things to be out of sync when the playbook was run for the `mpl-sphinx-theme` addition. --- matplotlib.org.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/matplotlib.org.yml b/matplotlib.org.yml index 86b10d5..c2b28b5 100644 --- a/matplotlib.org.yml +++ b/matplotlib.org.yml @@ -164,7 +164,10 @@ ansible.builtin.git: repo: "https://github.com/matplotlib/{{ item }}" dest: "/usr/share/caddy/{{ item }}" - version: gh-pages + version: >- + {{ + (item == 'matplotlib.github.com') | ternary('main', 'gh-pages') + }} loop: "{{ repos }}" # Caddy server setup From d6f452d68b9e0cc2849d4ced852364a882893663 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 22 Sep 2022 21:43:45 -0400 Subject: [PATCH 13/31] Hide mpl-altair with a redirect to GitHub --- matplotlib.org.yml | 1 - templates/Caddyfile.j2 | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/matplotlib.org.yml b/matplotlib.org.yml index c2b28b5..f552907 100644 --- a/matplotlib.org.yml +++ b/matplotlib.org.yml @@ -17,7 +17,6 @@ - governance - ipympl - matplotblog - - mpl-altair - mpl-bench - mpl-gui - mpl-sphinx-theme diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index d8b4b61..ef2f5c8 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -74,6 +74,10 @@ http://{{ caddy.addresses.main }}, http://{{ ansible_fqdn }} { import subproject {{ site }} {% endfor %} + # Hide mpl-altair until the site is fixed. + redir /mpl-altair https://github.com/matplotlib/mpl-altair temporary + redir /mpl-altair/* https://github.com/matplotlib/mpl-altair temporary + # redirect the objects.inv redir /objects.inv /stable/objects.inv permanent From 9e6ae8283033c3a6cafa75f6d235c84eb2ae0cbf Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sat, 8 Oct 2022 04:21:31 -0400 Subject: [PATCH 14/31] Make search go to stable docs instead of the mostly-empty brochure site --- templates/Caddyfile.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index ef2f5c8..f2929da 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -80,6 +80,8 @@ http://{{ caddy.addresses.main }}, http://{{ ansible_fqdn }} { # redirect the objects.inv redir /objects.inv /stable/objects.inv permanent + # Make search go to the actual docs instead of the mostly-empty brochure site. + redir /search.html /stable/search.html?{query} temporary # Place the brochure site at the top level. @brochure file { From 78520f1f0036250fea677731175e98169d1dfc20 Mon Sep 17 00:00:00 2001 From: hannah Date: Fri, 28 Oct 2022 01:45:35 -0400 Subject: [PATCH 15/31] add data-prototype built docs on https://github.com/matplotlib/data-prototype/issues are inaccessible because the redirect kicks but it's not listed on the redirect --- matplotlib.org.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/matplotlib.org.yml b/matplotlib.org.yml index f552907..428fc06 100644 --- a/matplotlib.org.yml +++ b/matplotlib.org.yml @@ -22,6 +22,7 @@ - mpl-sphinx-theme - mpl-third-party - pytest-mpl + - data-prototype vars_files: - files/webhook_vars.yml tasks: From 777f5df6fde7ee35a138379bb6f5d7122d6b7418 Mon Sep 17 00:00:00 2001 From: hannah Date: Fri, 28 Oct 2022 01:48:02 -0400 Subject: [PATCH 16/31] added page to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d83ffe4..ca91e52 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ related projects are also exposed as toplevel directories: * [mpl-gui](https://github.com/matplotlib/mpl-gui) * [mpl-sphinx-theme](https://github.com/matplotlib/mpl-sphinx-theme) * [mpl-third-party](https://github.com/matplotlib/mpl-third-party) +* [data-prototype](https://github.com/matplotlib/data-prototype) Ansible configuration ===================== From 3e2b42d543be752c051d88af8b447cd8aea208fa Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Sat, 29 Oct 2022 05:37:02 -0400 Subject: [PATCH 17/31] Add webhook secret for data-prototype --- files/webhook_vars.yml | 97 ++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 47 deletions(-) diff --git a/files/webhook_vars.yml b/files/webhook_vars.yml index 19f1b83..9b4c3d0 100644 --- a/files/webhook_vars.yml +++ b/files/webhook_vars.yml @@ -1,48 +1,51 @@ $ANSIBLE_VAULT;1.1;AES256 -62303938313862636161613161643430303838323031396332346336373531666361313731363737 -3365333464376237326365346336653561626435366530310a373936386439656533623238623638 -66393435653835363535646438336234356331306163646235316239633538366639353135376635 -6234326362333662630a303830663565623061363561636330333363393130653266306339303631 -62363862623934306539323765636466336238643664626237326362366434353166663963383534 -35356663373533363863376534386532393732623066393832366436323630393265323436393064 -64303439323363306665323761306262623663636661333134643933366262306136643037313262 -38613437343266353464663964343666653366356563323934353164376266626463353864353336 -63323065626166373062666632616332643338323638303261633363633935316637653434356632 -63653964386262386363616662353634643836616565613465613033366237343031616430663766 -63666439343638323262343936373137653239626438626565333336383365343832626661383663 -39356163346638323630343262646261333862313730623431663164303435393837626231326537 -34313733346261623530663762373731643332396664383061643232393638346631373734393230 -31396565633531636439626562643466303930646562323734613139613865653266313837366663 -35353365613764623530313430363465643164643435636464633165313066653961363432613864 -31353665613830333866356132633464633937313231376230653631633133376161653861343463 -65613630633236613531373262633936373934343735383431306337376562323933353963633739 -31313736326238313232626532353331333230656665373633663863653065386536306435363237 -31313936366130303134333036663764303665383735663130303963653832303538633339316561 -39326634633166613934333937663430626433613961353831343934663538323835373365633166 -31366135333536323734343936653162636562376234616635363136643530636166653337313435 -31343632646564616666653162353064366336633239633034636563623336313034313337316335 -38316566343938623265336662643234653964373034306635636135353339306535663962356130 -62616663396339333531313061323732376365663565616663313465653062303866383235353832 -36366531643064303731353230323634376538306139346364633662626261386638343632383030 -30333364363530373131393765396533303431643939636361643231313437666334323532346162 -37353538623233643738366135376339336635623132653337376563353334383362363066343863 -39663462316330663062336232663638623062333432643566386233333963373938623864666439 -61643637333638633734353464333733343735303737323864663733316265333865623864373963 -36386262633036393734353439623434306139303331353464633637663261646438306137613638 -31336463363738643665383461333432643066383961663638353133626236393738626637363533 -34656666346630346631663833333837656130666333666230653136363634386233353262376537 -36333233306561666133346537396639306332336136363738626635363466346132326463356163 -32366166663336343262313933333162353630623064623338633265313464393636623936633033 -61646237366361303164663663313139386432343533666666393131353631626330383961666134 -66653630306332633438323336306363333737646334356465646136613235353362626161633064 -32333931376530666263363863333661333938653237353934303439336334353630353262636435 -39373536396465666165643261303237636364303138616239636438353864386634346138303738 -34626130356262633833373231353066653837303963303930623931363435313537383539366432 -34376661653963343964346663336533346565373435366663363831636338363230373862366235 -35323431653830656634376636373033356337626533353766386534363465343634613161396366 -38386131616534326135646134353832393765393834656239663464306530613937346266396232 -31626561626530613761323665363833653037633761313438383139303565653131386264613963 -31666166383738633938663536643864303866656666333439386133323634646537653865613138 -64373339373338383065666235306665663238653666393361313837333832373334626433663633 -36373765346165613033643734626364663334633433343865616539623564353630313663636338 -663137626238383764616238333262343630 +33343236653933356235316138643738663638663431633834333132356664343565633038626637 +3032336139616233366566326331326433396631363936340a323465616236366263633830373566 +34623662663638653836336165396637623763626238626232306339303035613863613431663663 +6139633863663937340a316464613234366135646565653066343665613335303733373239343464 +39323339363863646238383530633036663739333131643766313061333037326561386239663230 +30663139333162363366636231643038373030616438336230623431666361336436363430346365 +32303230643936666163633838343465643333303337616663343031343066303332376239623566 +64393838323166646463393664346132653831386664386162663430656264656130323730366663 +62333566353638383132633638653932323761366235306235356535313434343535666435326338 +31356639313064653338376366356532383734663134336563623336396439353038386638383362 +39383931633232636336393936613837396336333236343765626338666434363539643961346230 +61363061626430633431363439653265323038363361306135363164663138303839386563313330 +63623661386337353638643462366435356564653031623464663434306132653733323133366530 +66633761653731663833316438323764363830353834303066633637663361383839343464626537 +35303661366536383665353937636130303635616232313935646662633735313366663832343138 +66653865626162623736306330626162303132393539616664353266326662303766383266333533 +36396664343763633466396664383436316165336565656634356261386562613237663530633438 +32393532333731363036343537363136613830616536366462386534616339363063626131303639 +33353437383666353231356138366535646238396538353139666339623535326566386635383762 +38613134353861376232666139363438316262346637346339663261366434363261636336323030 +37663438663430626634666236313235383532393435643938336334396262643562373739623431 +64623231663761643931663164656461613730623833633665316365616132363361326138333961 +63386531666333343234393366333833326463326231343935303636643335373237333035616363 +36303565633564343366346237643034636137653032383736633866323731323264393562396638 +65356133623435363666373765626232386639386266626534356131623435316139613934376436 +30616266393536353233343561653762353661646537303035663763313234346638326466323539 +38663063633764303430356463623863663238633861653734333339623339613465366436393637 +65623366643539306563303030616536663637326534653036366637363937386132623664313330 +39333065653963323965653565313762343638383930643730333462366665383137313961313137 +62363762383665613837383432373663353736643766306630313934303964633337316363636163 +61373531626466396139343532653938626362343436663139646261643966353461326364613465 +64333366383861353161653439653164393631633734326638363064613363366163356661373661 +37383365663732643337373539333933363834393663623963343730333838623562306335303536 +31393636363133353936353237623264356464623965663562323366633862353837623466613835 +34623863386231656464306165343933336631303462643734386631343530653731393737663662 +36303463656630396366316238656466383765383737363232316333303736333439363762373836 +35643030656462623866396164623934643132653666633034623631356131643536313830356532 +37373331323637393563373631616139313261303731623265626330666261663230623265303335 +39346265376436396530623638316235386638353831373164356532376339326237343365346338 +37393661613263626232306265626433303262663535653735343637316665316138323631663335 +30306438623739316132653032333365613266336461343162333265663065653133373337363165 +62623931343437636263333930363038383836343731336430663061396338646565613064373665 +34356263396139346631633936383339356335306530373265656236613934653162316562633666 +61396562303961663762323531643962636564626265376661376232333064353363373062376137 +64666565333637613032323762623639353236316636623262623065363165363138356432636534 +33623436613935363533636131616630643031306564333037323731393733643132643239356337 +36383961303166393430383432646561306165303637323761653131306237353862323063623565 +30343334613133383839616635396339636436343736656639613561396330306533623663326239 +61623765386237653366643864343131346262353764626363353736333238643136333630653037 +3636313661656638366234363566653037386131326138343038 From 6fdba765f81b4fc23aa0613f5490d1158b75b14e Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 20 Apr 2023 16:01:12 -0400 Subject: [PATCH 18/31] Update TLS certificates --- files/tls-cert.pem | 124 +++++++++++++++++++++--------------------- files/tls-privkey.pem | 34 ++++++------ 2 files changed, 79 insertions(+), 79 deletions(-) diff --git a/files/tls-cert.pem b/files/tls-cert.pem index 6d72019..63470f7 100644 --- a/files/tls-cert.pem +++ b/files/tls-cert.pem @@ -1,63 +1,63 @@ $ANSIBLE_VAULT;1.1;AES256 -32363734336562313837323335393033323465643861643535343331376534663062326333613963 -6335316335636434656336643830616135306561666561650a343235633630396637383337656238 -63343064626165613432346438376664333537336539643962616636343936633332663438313532 -3064663335356361390a303033356334383836653163383533626637363365323031643165333962 -37303761393237663830333532383230613331306630636237326665366264616665626431363235 -33336230663639353731356266316234363961613331633231333766303833626665633336376634 -36373831343462373733393262386636323365626338316538393330323734306261383235393961 -39316634396161393539346361383238663935663064376131383535363836326238363466653261 -39616365313862396262336336306631623838373166616339373435393538623339313263333033 -66646662313436356365636665303234313464303763306331613064323734646431663165333862 -39396366646335613535613463313863373136336434656534376565313337346264386533333430 -34356137396461386161393437346637316231393964313534363036623031383339366334373631 -38383264366438306435326237393130336133326361393335386361323732396239613464623864 -35346330343834333739333964343634396238306665323462613561396262373435336365383134 -66323066383436386437323864383630383238356564363030306639326538636565386563353665 -33633432326437623233386133313537613837663739353239623966613531646338623264386131 -63303131626530376137626436316561306265653262646536313662303665333334353238313065 -64303336303134656333646466613261663137636334633039336636643938333365303136353362 -38326633653236326138383263346532653639613366646136643537646535393262646462633236 -33656238616165323433313033386564636261316236363039663335343561333166643565383739 -32623664613465356137313730343465336664393832626131386331393536393435623062343130 -38343832303830643862383935356635363766643262386237343336623336643936323463666439 -61386566636432323166386239323164373133323333653265376234363532386564333530613332 -34633465666336663638626466626433383834623236646566303031663365636161383137626361 -61316261303265306161653033653937663938616439656163373231396163623037333361323530 -30663365636433306561316165393832396162646335636266306534636634356134376334393331 -39643137333764663261323566353030343562316161316535343361316335613431353834663866 -32643537366432323635653132346163303034656232313165313239393465656666323530656663 -34663064303734363735316338636233613661303234626339323130633035663834613232663333 -33383734313133363763386566633933386132613437616163353536653963353031333466626133 -37396664303337313561626136666363343133656161356239393236346163343834653536306337 -34623239653734353164366633313537353434636264393236626531376132626262623466663830 -37383731623130346365383336336437373838613931663130656365323831326138643664663366 -36303831653066373566363562343830363830333766613465353033373634313234326366313239 -36353339393862653734343135623932323639613864656637393033343339356437626161663935 -66353437373934646138386232363261333166636133613136613262326639643334646362353035 -64386533633835626536346233343762636435626161383535343631636138316462626438616232 -61653333386561613834666462333533333666613730633132633233353335343536643232333633 -32383561346461653030633861313930336536343363623833343066373535616365346264666436 -66373663633432623666313265643035356333393237353532393436643432383964313937303466 -34663436653864643334386265316133656134613532313034633430356530626536626132376136 -35383130373931663739636365373666373333313865663437616161333039373532306637643036 -38303534313734366237356432346439346334373530333332386236323165633336336331356166 -30333332666566623631373333333132653466396636316332613064373938356238663439666665 -37386235623830396132363236306133316538613039626636663537646137326461373161666630 -32323366343838316539623163363738316165623531633933323738373239396265643330396361 -37353162636639316234393336663737623930663563313464363664326130656537323265306138 -63646561626439313834353632383761623837636163356431666237333633333265383337643436 -33393939326362323933323134386466626534366261366236366430383834326466343435333161 -31653531323264663463623230326363623730356339356636643930653164396263623131323533 -64313265333832343463353961383966653433626661666231326537643362306330313631393661 -30383264393766386232303964306135636437323464613332646230633636623934633139306266 -63346237366233356261336333353864626262663766396461393762633162336239333738346437 -65646630363637613565366365363433383835643732356339346234646465326135343937383731 -36323334336663363537396564303131386539303937373639623965643033366666376230643861 -34313563393130613164353462636466363933313261666231333630613532366232383866326563 -30353732383066343866616363343063643665663564663331356235373064323062643230633665 -65313365633863383663633235376233386562353263383965376566333161653962326635336463 -37336161353434333161656134623466643832323565653930646236353263346465303132643939 -39653832626530366238363934313662313438393836653932326536386533326437616631316537 -32643162663363383265323965616637303261626334343332613632646533356536646562303034 -6461663738393262636530653062623661623139353530323231 +64636339376633393063343733636561613464616631303930643133633130343939356262303561 +3562376230363432393332343064313864653635373030650a393833303864303136366230646434 +63646538383830656433333262373132633466366134663431363965646664316335303063653461 +3135663531336264310a363931646561383964646130383138333830323235316565396638643530 +36353537313864376436633665623736323638336663393333326131383930613664653961383634 +61636166333065656537356536306632386237313635303530633632376632376635396535643131 +33623865626437336433303535363564653365376638313036393564393333613333646136303862 +62393863363630616437346632636364313731333137643338376465353563636234333364363837 +34393164323832633837343133306365373735343231356161653365613066643062326433306432 +30663535326664316564376637643533326565633961656261653962633961363966323261393466 +36623464316130616261313363353466356335393031656435656638396635396362363838666365 +61333264663831376633316239616234666338316332393636646635653766366632383732653363 +33643137323937623764616333626263663039396561356166656238396231643130663263346662 +61356637383361383034306337396233383637363632326138613666313234396231616334333130 +61316537633061633033653432373933323639336538313736373161643663393564343562396539 +63366539386234363236306139633161306661343239663238396163316161363861356532386531 +34343165653831346539643732646636643731623330353563343333653833373237363738623865 +39336532643631323531333935363566623134613930363032383134333632346135373761653964 +64623135666633646139623863666337643265636435643834346336616363363431306262643338 +30343732666364346463353061303631396637376537613935313263653561623464323961356336 +38366366623430333966313830313437353866363831343938303834393237363764316430313464 +65356466653665626434363630303438376262313165643636303432303737663864396434333064 +35613530396262343361363832343463623564623733636538333435613662633963613837313835 +66303861373832306339303232323238383439376636356133633264353132323233623838343632 +64383230633965643464653334303531313264623739363334396233633162323938393533626563 +62333835633261613036316534656235633664323238323233366234396133343430663365623161 +38383765333231633731653235666365643931383662396361336564383039326431643661626665 +61623736633965303062653865333236626331343632646135656238666433313439663465623364 +35366364393037393563323663633762383834613336316236313935363238313861363661623233 +61333238623661643661376332393765333363383336383465366536353436313131616433353431 +34626238386438646532663538343936656330346636633832616166323833666265303637386337 +31666138643163613839336266666531316632363533386462623731653737616430666431396132 +30643733653234653965383030313064663434616536633165386563663538363430646166393362 +66666539666261333938356433306332306463623330623832616530346364306235326261626435 +39623636366431316133646366613736363564623032303033656339616566663136346435643934 +65613763616535353265343262333736393133646661616264656631353932326561333931643430 +36346461316136353832623537626664666636366562653836353265363139623364633833373031 +30663932666437333936386163643461313538313833656138333033373735623632333964636563 +30316531666663356263313031383937353231353839626233653161636237303365386430633365 +38323338393533313438373237623562623261626132393937303335373465643536303935626634 +32346431373530346636666532626138313838336532666363663234613961383161306333313361 +38383466393961636637323664653637336630623962303737656232306537613337633338366164 +35653331626661663137373635336139346535623235396463313761643234356431313062323530 +66663639386236323362626664373264643638343737396434653939346262653736303332306261 +65646466333436663735663066656161633861373030386664326565326164656136343630616533 +61353166393464363664613930353437653438323563303031633433666562383365356436383537 +35323232663935383730626132323365633566383938393236303162326666663832643433336363 +30343037353965623738356265313934346666646332366666643264663239373633313334313239 +63353464383639323661353735643238663636386563623336376465353931383933393139373166 +62653131363364323833366364366562356638306530373539646330646331653331633564656333 +37653336306461353064323563646163663865303566363232326336373536313461353737323164 +33376661366534333537363731383561376366623366306263316131363066396230343037326636 +37353135333862316330393537613264613861666166633834323033616463616235633039613565 +34303933363530376639333365336538363932356134636131386231333961393339333134316438 +38653834323937666433343833653536323533363839373535393238623033656336386332396364 +33326566393830663337303035653065336262333263313665373034353634336631353535356139 +31643965643238373163376539366436333461613439353833616634623263363738336531326138 +38343438383461386136643732343132363634333961393831643363306235336265303238343339 +31623462613033303337623731336564643666663537313365383730326337323262343836383363 +36666663643066643934396137346366383262666431303137633031623234643561656661303365 +31663539363464313636393835646663626234383061626566383730666265643032626633623936 +6436313331343332393464616464386338613861626561623161 diff --git a/files/tls-privkey.pem b/files/tls-privkey.pem index 2f0b0bc..58b444a 100644 --- a/files/tls-privkey.pem +++ b/files/tls-privkey.pem @@ -1,18 +1,18 @@ $ANSIBLE_VAULT;1.1;AES256 -32623665323834396639373763366430313561306530373663363537626637646239343531326661 -3638616465343365313038393865303266356131383862310a666633633762623534653734326533 -32643364666631353830313639616230616433393562626132393233356462303736323732326562 -3937373866313830650a643463353930323136393664643230366665303337633238666464376436 -39643233353738333936633237653939663735623366323331616335393735643230343137383131 -34326462663866326261353337373830643839343662666665613737616332306330373638393735 -39653535356261393266303636373332646135373362646531343032653438303730363838393837 -36376363633361636166373234663331323736646364656634393230616138656165343235383036 -30346632333438663165333035623634353436653030396330623862393066393265353661353963 -65353033313239333762346161313238313537343632643262656139653934363836666336613739 -33336366386136626131313135633463303561396262343333323164333264376133616131366664 -36386366653137656336623230316531333530303061663461383065383361346562343164383134 -64663633313533623333633761616137393631336161643035626238623735663138326266363339 -62623033313032623637623335643134313733306435313235646634326663663036343564323461 -37636563626463656263656434666131343166303137333763633966346430666565346535323539 -30383038353138353936646338626332633738633632363432393436613633363431326533313830 -3664 +61303034396434383432343035386165326236616263333131346565346337343566303039373062 +3138373432333164306365656461633665376165353263310a343966306335313433663538643836 +62306133656666653035346161353464653961653638353639616363326436336439656634663633 +3562373161333362370a313730616537393565323064383666396134343265353165643566393437 +66393063366166396662313437323937363237623262333531376565323563303839353961666337 +30393337643261366438336138386635303636643536633233336334626665646337613332646330 +31343466633932356430666563643962363465623839326563336139316364623266626132323062 +66323339663164306538313239646633636165363365616337363564383966616639303630636462 +64633330636463633265346330383566333234633630636566383030373061663366363131333861 +61376330326233666136393138366436303339343730633665346134613663303566333762396533 +31383138313030623636393562356133386463356262613766373536343235326334656362356237 +61646630343534633733613333613165356366383235656439356364636537656462373061653961 +35343932643037646464336232646466323833633838643337613933366466316636323665663633 +33313630633034303134663139666133613439643132636562616334303930396233323861623130 +35306665366162353537326335636361333138353063373532313964633337613666373465653035 +66643766373363353334383763323235656363643832633031316335306433626136393763386231 +3763 From d95587918acf92344e96964865ee496d62364477 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 30 Nov 2023 21:27:56 -0500 Subject: [PATCH 19/31] Redirect development guides from /stable to /devdocs --- templates/Caddyfile.j2 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index f2929da..37ba2d5 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -83,6 +83,19 @@ http://{{ caddy.addresses.main }}, http://{{ ansible_fqdn }} { # Make search go to the actual docs instead of the mostly-empty brochure site. redir /search.html /stable/search.html?{query} temporary + # Redirect contributing guides to latest version. + # If you go to /devel/...?reallystable=1 you can avoid the redirect. + @devel { + path /stable/devel /stable/devel/* + not query reallystable=1 + } + handle @devel { + route { + uri strip_prefix /stable + redir * /devdocs{uri} + } + } + # Place the brochure site at the top level. @brochure file { root {{ caddy.site_dir }}/mpl-brochure-site From e272779cd07bfcaf37f2cf4b57231ec49d0a8285 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Tue, 19 Dec 2023 17:36:29 -0500 Subject: [PATCH 20/31] Add an Ansible playbook for creating a droplet --- README.md | 70 ++++++----------- collections/requirements.yml | 1 + create.yml | 142 +++++++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 45 deletions(-) create mode 100644 create.yml diff --git a/README.md b/README.md index ca91e52..1c9e883 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,9 @@ prerequisites: * Create a DigitalOcean API token, and pass it to the inventory generator by setting the `DO_API_TOKEN` environment variable. +* If you are creating a new droplet, and want to configure DNS as well, then + create a CloudFlare API token, and pass it to the Ansible playbook by setting + the `CLOUDFLARE_TOKEN` environment variable. * Set the vault decryption password of the Ansible vaulted file with our secrets. This may be done by setting the `ANSIBLE_VAULT_PASSWORD_FILE` environment variable to point to a file containing the password. @@ -99,9 +102,11 @@ Naming We follow a simplified version of the naming scheme on [this blog post](https://mnx.io/blog/a-proper-server-naming-scheme/): -* Servers are named `.matplotlib.org` in A records. -* Servers get a functional CNAME alias (e.g., `web01.matplotlib.org`). -* matplotlib.org is a CNAME to the functional CNAME of a server. +* Servers are named `.matplotlib.org` in A records, pointing to the + IPv4 address of the droplet. +* Servers get a functional CNAME alias (e.g., `web01.matplotlib.org`) pointing + to the hostname `.matplotlib.org`. +* matplotlib.org is a CNAME alias of the functional CNAME of a server. We use [planets in our Solar System](https://namingschemes.com/Solar_System) for the name prefix. When creating a new server, pick the next one in the list. @@ -113,51 +118,34 @@ The summary of the initial setup is: 1. Create the droplet with monitoring and relevant SSH keys. 2. Assign new droplet to the matplotlib.org project and the Web firewall. -3. Grab the SSH host fingerprints. -4. Reboot. +3. Add DNS entries pointing to the server on CloudFlare. +4. Grab the SSH host fingerprints. +5. Reboot. -We currently use a simple $10 droplet from DigitalOcean. You can create one -from the control panel, or using the `doctl` utility. Be sure to enable -monitoring, and add the `website` tag and relevant SSH keys to the droplet. An -example of using `doctl` is the following: +We currently use a simple $12 droplet from DigitalOcean. You can create one +from the control panel, or using the `create.yml` Ansible playbook: ``` -doctl compute droplet create \ - --image fedora-35-x64 \ - --region tor1 \ - --size s-1vcpu-2gb \ - --ssh-keys , \ - --tag-name website \ - --enable-monitoring \ - venus.matplotlib.org +ansible-playbook create.yml ``` -Note, you will have to use `doctl compute ssh-key list` to get the IDs of the -relevant SSH keys saved on DigitalOcean, and substitute them above. Save the ID -of the new droplet from the output, e.g., in: +This playbook will prompt you for 3 settings: -``` -ID Name Public IPv4 Private IPv4 Public IPv6 Memory VCPUs Disk Region Image VPC UUID Status Tags Features Volumes -294098687 mpl.org 2048 1 50 tor1 Fedora 35 x64 new website monitoring,droplet_agent -``` - -the droplet ID is 294098687. +1. The host name of the droplet, which should follow the naming convention + above. +2. The functional CNAME alias of the droplet. +3. The names of SSH keys to add to the droplet. - -You should also assign the new droplet to the `matplotlib.org` project and the -`Web` firewall: +You may also pass these directly to Ansible as: ``` -doctl projects list -# Get ID of the matplotlib.org project from the output. -doctl projects resources assign --resource=do:droplet: - - -doctl compute firewall list -# Get ID of the Web firewall from the output. -doctl compute firewall add-droplets --droplet-ids +ansible-playbook create.yml --extra-vars "host=pluto functional=web99 ssh_keys='a b c'" ``` +The playbook will create the server, as well as add DNS records on CloudFlare. +Note, you must set `DO_API_TOKEN` and `CLOUDFLARE_TOKEN` in the environment to +access these services. + Then, to ensure you are connecting to the expected server, you should grab the SSH host keys via the DigitalOcean Droplet Console: @@ -181,14 +169,6 @@ Finally, you should reboot the droplet. This is due to a bug in cloud-init on DigitalOcean, which generates a new machine ID after startup, causing system logs to be seem invisible. -DNS setup ---------- - -1. Add an A record for `.matplotlib.org` to the IPv4 address of the new - droplet. -2. Add a CNAME record for `webNN.matplotlib.org` pointing to the given - ``. - Running Ansible --------------- diff --git a/collections/requirements.yml b/collections/requirements.yml index cb8953c..4a795f3 100644 --- a/collections/requirements.yml +++ b/collections/requirements.yml @@ -2,4 +2,5 @@ collections: - name: ansible.posix - name: community.general + version: ">=2.0.0" - name: community.digitalocean diff --git a/create.yml b/create.yml new file mode 100644 index 0000000..f23dbdd --- /dev/null +++ b/create.yml @@ -0,0 +1,142 @@ +--- +- hosts: localhost + tasks: + - name: Gather information about DigitalOcean droplets + community.digitalocean.digital_ocean_droplet_info: + register: do_droplets + - name: Gather information about DigitalOcean SSH keys + community.digitalocean.digital_ocean_sshkey_info: + register: do_ssh_keys + + - name: Print info on existing droplets + ansible.builtin.debug: + msg: >- + {{ item.name }}: + {{ item.networks.v4 | map(attribute='ip_address') | join(',') }} + loop: "{{ do_droplets.data }}" + loop_control: + label: "{{ item.id }}" + + - name: "Enter name for new droplet (subdomain only)" + ansible.builtin.pause: + register: input_name + when: host is not defined + + - name: "Enter functional name for new droplet (webNN)" + ansible.builtin.pause: + register: input_functional + when: functional is not defined + + - name: Print available SSH public keys + ansible.builtin.debug: + msg: "{{ item.name}} {{ item.fingerprint }}" + loop: "{{ do_ssh_keys.data }}" + loop_control: + label: "{{ item.id }}" + + - name: "Enter SSH key names for new droplet (space separated)" + ansible.builtin.pause: + register: input_ssh_keys + when: ssh_keys is not defined + + - name: Set droplet facts + ansible.builtin.set_fact: + host: >- + {{ + (host if host is defined else input_name.user_input) | + trim + }} + functional: >- + {{ + (functional if functional is defined else input_functional.user_input) | + trim + }} + ssh_fingerprints: >- + {{ + do_ssh_keys.data | + selectattr( + 'name', + 'in', + (ssh_keys if ssh_keys is defined + else input_ssh_keys.user_input) | split) | + map(attribute='fingerprint') + }} + + - name: Verify droplet configuration + ansible.builtin.assert: + that: + - host in valid_planets + # Must not be an existing name. + - >- + do_droplets.data | + selectattr('name', 'equalto', '{{ host }}.matplotlib.org') | + count == 0 + # TODO: Also check that functional name doesn't already exist. + - functional is regex('^web[0-9][0-9]$') + # At least 1 key, and same number as requested. + - ssh_fingerprints | length >= 1 + - >- + ssh_fingerprints | length == ( + ssh_keys if ssh_keys is defined + else input_ssh_keys.user_input) | split | length + + - name: Print configuration + ansible.builtin.debug: + msg: "Creating droplet '{{ host }}' with SSH keys {{ ssh_fingerprints }}" + + - name: Please verify the above configuration + ansible.builtin.pause: + + - name: Create droplet on DigitalOcean + community.digitalocean.digital_ocean_droplet: + state: present + name: "{{ host }}.matplotlib.org" + firewall: + - Web + image: fedora-39-x64 + monitoring: true + project: matplotlib.org + region: tor1 + size: s-1vcpu-2gb + ssh_keys: "{{ ssh_fingerprints }}" + tags: + - website + unique_name: true + register: new_droplet + + - name: Setup DNS for droplet on CloudFlare + community.general.cloudflare_dns: + state: present + proxied: true + record: "{{ host }}" + type: A + value: >- + {{ + new_droplet.data.droplet.networks.v4 | + selectattr('type', 'equalto', 'public') | + map(attribute='ip_address') | + first + }} + zone: matplotlib.org + + - name: Setup functional DNS for droplet on CloudFlare + community.general.cloudflare_dns: + state: present + proxied: true + record: "{{ functional }}" + type: CNAME + value: "{{ host }}.matplotlib.org" + zone: matplotlib.org + + vars: + # We currently name servers based on planets in the Solar System. + valid_planets: + - mercury + - venus + - earth + - mars + - jupiter + - saturn + - uranus + - neptune + - pluto From 643515fe70af8c8d2f2f7a7a1bce1715a9b1fd54 Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 15 May 2024 15:26:23 -0400 Subject: [PATCH 21/31] MNT: update DO <-> CF certs --- files/tls-cert.pem | 150 +++++++++++++++++++++++++----------------- files/tls-privkey.pem | 106 ++++++++++++++++++++++++----- 2 files changed, 177 insertions(+), 79 deletions(-) diff --git a/files/tls-cert.pem b/files/tls-cert.pem index 63470f7..aa36f03 100644 --- a/files/tls-cert.pem +++ b/files/tls-cert.pem @@ -1,63 +1,89 @@ $ANSIBLE_VAULT;1.1;AES256 -64636339376633393063343733636561613464616631303930643133633130343939356262303561 -3562376230363432393332343064313864653635373030650a393833303864303136366230646434 -63646538383830656433333262373132633466366134663431363965646664316335303063653461 -3135663531336264310a363931646561383964646130383138333830323235316565396638643530 -36353537313864376436633665623736323638336663393333326131383930613664653961383634 -61636166333065656537356536306632386237313635303530633632376632376635396535643131 -33623865626437336433303535363564653365376638313036393564393333613333646136303862 -62393863363630616437346632636364313731333137643338376465353563636234333364363837 -34393164323832633837343133306365373735343231356161653365613066643062326433306432 -30663535326664316564376637643533326565633961656261653962633961363966323261393466 -36623464316130616261313363353466356335393031656435656638396635396362363838666365 -61333264663831376633316239616234666338316332393636646635653766366632383732653363 -33643137323937623764616333626263663039396561356166656238396231643130663263346662 -61356637383361383034306337396233383637363632326138613666313234396231616334333130 -61316537633061633033653432373933323639336538313736373161643663393564343562396539 -63366539386234363236306139633161306661343239663238396163316161363861356532386531 -34343165653831346539643732646636643731623330353563343333653833373237363738623865 -39336532643631323531333935363566623134613930363032383134333632346135373761653964 -64623135666633646139623863666337643265636435643834346336616363363431306262643338 -30343732666364346463353061303631396637376537613935313263653561623464323961356336 -38366366623430333966313830313437353866363831343938303834393237363764316430313464 -65356466653665626434363630303438376262313165643636303432303737663864396434333064 -35613530396262343361363832343463623564623733636538333435613662633963613837313835 -66303861373832306339303232323238383439376636356133633264353132323233623838343632 -64383230633965643464653334303531313264623739363334396233633162323938393533626563 -62333835633261613036316534656235633664323238323233366234396133343430663365623161 -38383765333231633731653235666365643931383662396361336564383039326431643661626665 -61623736633965303062653865333236626331343632646135656238666433313439663465623364 -35366364393037393563323663633762383834613336316236313935363238313861363661623233 -61333238623661643661376332393765333363383336383465366536353436313131616433353431 -34626238386438646532663538343936656330346636633832616166323833666265303637386337 -31666138643163613839336266666531316632363533386462623731653737616430666431396132 -30643733653234653965383030313064663434616536633165386563663538363430646166393362 -66666539666261333938356433306332306463623330623832616530346364306235326261626435 -39623636366431316133646366613736363564623032303033656339616566663136346435643934 -65613763616535353265343262333736393133646661616264656631353932326561333931643430 -36346461316136353832623537626664666636366562653836353265363139623364633833373031 -30663932666437333936386163643461313538313833656138333033373735623632333964636563 -30316531666663356263313031383937353231353839626233653161636237303365386430633365 -38323338393533313438373237623562623261626132393937303335373465643536303935626634 -32346431373530346636666532626138313838336532666363663234613961383161306333313361 -38383466393961636637323664653637336630623962303737656232306537613337633338366164 -35653331626661663137373635336139346535623235396463313761643234356431313062323530 -66663639386236323362626664373264643638343737396434653939346262653736303332306261 -65646466333436663735663066656161633861373030386664326565326164656136343630616533 -61353166393464363664613930353437653438323563303031633433666562383365356436383537 -35323232663935383730626132323365633566383938393236303162326666663832643433336363 -30343037353965623738356265313934346666646332366666643264663239373633313334313239 -63353464383639323661353735643238663636386563623336376465353931383933393139373166 -62653131363364323833366364366562356638306530373539646330646331653331633564656333 -37653336306461353064323563646163663865303566363232326336373536313461353737323164 -33376661366534333537363731383561376366623366306263316131363066396230343037326636 -37353135333862316330393537613264613861666166633834323033616463616235633039613565 -34303933363530376639333365336538363932356134636131386231333961393339333134316438 -38653834323937666433343833653536323533363839373535393238623033656336386332396364 -33326566393830663337303035653065336262333263313665373034353634336631353535356139 -31643965643238373163376539366436333461613439353833616634623263363738336531326138 -38343438383461386136643732343132363634333961393831643363306235336265303238343339 -31623462613033303337623731336564643666663537313365383730326337323262343836383363 -36666663643066643934396137346366383262666431303137633031623234643561656661303365 -31663539363464313636393835646663626234383061626566383730666265643032626633623936 -6436313331343332393464616464386338613861626561623161 +65613739323638356330343161393831666139646464303834366437316162663761633237623136 +3866643839616536376133303963613964356436386462350a363238393266663064376338356232 +36653239383865303265346364616531363638383366356132656162323735633931356463383530 +3266363935666330660a616534383530616337616538383935333631633731663563316436356465 +36623831623335303738316262303762346364343962396566303436636236623061316433653639 +65656337636664633064363562333763363062623139616637636237383566666431306435623366 +33643463336137653663633166626161616265613265363535343066393835616636383432613439 +30333738383461333436333133663131363432383130353939623237663834613931343263663063 +63313338633436346435636336313233616130303030613633396539636431346339636135656361 +37363665303530346635316265666530653966383765663162303939616636326163343434353233 +31366265366635393032356664393339393134613630313338373735343530653463306333643764 +33376631616665343866393532386535313835396534353936333262653332346335353539666663 +66363134383030643631376134343235383964373738646363363365306564353937663738303863 +32646232633365363439613531666139653562363932333536326131313630643361316464656633 +35316234326338313132346537636631346166613761343134663661373266396561323430356166 +33663530643465343432383034356638656466656536336563363630333565633965376663636235 +37313735616434656664316364336538326633346638653032373637313033353338343666323438 +64376231613039356563313266393837663166653836343334636238333265326432653761373538 +62303839613732393633383732376533613939366664386637363437623630666535363038346562 +65613762636531353437356564383263366666363035653131653062316437653234653765623163 +32323233346436646335626535656166336139653034353030356330333165393132636566373764 +63393664663331346666343361646233353666633834323762656632313661663038363166616363 +32333664633437303332363931313166393137383032333038383834613161663861663464393831 +30643839376139646338366339373766666364316338616536633864613632333436343139613066 +31663062323964306231336163666138636362643232643239663964383164636338363837383638 +36643066643762383836636631306436366662323065326163656538643438383663623466353630 +34353562646331313731653533313232323236313264373366666161376630633633373962616562 +61353931376463303162336238303030313235616466386638636661303035323433653932383033 +63653862306263366663343839383438373836636535396333386637383032636365353735383166 +62623030353730346364336566653961356131653739653161326665613730383533363861366264 +39373963653732303263303563626163616535396237306163653534386565323231633030653238 +36323936306238313365646439633433623133366466633635373363623839303135306466646336 +65356365653532386132306230356163323234313538356132643932656334303165343765356436 +38633666353730633039383666313636656234343932396138323534383134626363396263616338 +36613462383139656132396534653030313232383865623532613763626534373865666533623766 +31346364316566356133366336396633356637333035376437653339663661386130653162386466 +30643231613235646234376631616138353562623734333361616530623861343435363032303738 +61653365396464306563616461633961633261343639366631326232373436316462643130633830 +31396537623265366364393830303061303738393431393331333462346165373166396537643234 +63643834323165373664333239373761353362363932613166303331336332643536623037643737 +36356563306333316431303434633661636464356631373866633334616339646661373533663331 +36656334656534303835663638363631393862333164393532306364646239323636346137396230 +33313933663736323336336531303031663638316563333533303930356533373161663766623934 +63313039393637636532343130613135386536323035316136313737626239643339643139663864 +66363634363239383338323433666362323238643538366662656537363130353930303139643665 +32626439336633333661356538396164643461393039333832316662666537356439326236653139 +64393439303836316233316136666430383832623663346638303162386562373937613032666663 +63623538663964376663643837323163303734616134323263353031383832363939656565613237 +33653237343962373237656561323539643964313532656162613464613037633735623338356262 +65636438303466636539326232636133366533623863336230623030363532353766353639653531 +64633831326162353836653534313038643932313738393261383536383365323138383331616364 +66336164386436333634393732373530343036386237333230313238306465356139373030356637 +65636461343232343938373261326662343461653635363938616437353435346335393530393738 +31653463326530336337306438356565666161366433363766363265633037353437386534353732 +37323265396538376664356338303236633831313534623836643661623166613637323833376166 +35353031313139626562666363353561313832613865666461393864383338626434343735633836 +37663935376165613963326461616639623731356235353461346334376361303531316336383934 +63656162656662353133636539313337366136616637633935323965663635653864396332373635 +63353336623237376533366162303965343563616266616138613435376564616136376162386365 +35363237376432313765646434323830333765626636623663343933343038383930383136316334 +64336133353835336465313132643639366139346536313261346234346335316534396161393435 +35653631303337633264646430366364373863386533353535303736366433313231316262636161 +36663233303437343664363161663761356532396433616331306437663530366435653632356234 +30653830336562303234356635656338653030633030323434663161303533616565356431346131 +64643030396562623165613837343336653265316530633165626131363938336233356531373134 +61316432383735643335303639316136366138646565646236396336383463343062383430653933 +66616536383964646234616363613633666432323863666132623764326465383462336637303434 +35313834363834623630393636626665623639633538356532366238626163303431653733623136 +39333864356331626535353731346136316238333264646131616266636335643766613831383233 +32613662626137323138336637323934353439653434646432343434333661663164386439643665 +34316363373563626362323735306335613231343533376666643763326339386562663066336437 +33663065386562386634616439336665393663333032346664663364313761366262383337613565 +39383633663763313664316462646230373835626338386130623164356163383833336234343633 +61313834656662623132363735656238366561623237376264366638666534303165396565613731 +39373762316437613432343835383837613430623434626134383262656461663565663030366137 +30656265663936643666353365616637356335376539366639303731623665646232346439396166 +63646664306465626435366266323032666530346436326264323637643830666236376563303234 +61373436623464323333333165356335636632323039646130333237303839316434366532613632 +34623133376437306134373437353534326662313261323038643231366439343830333934356632 +63373463636231343931343931333031663239633436313739633937626539386166303164633934 +32386432313733393730623962623265366436653731333665663830303439636634613466303530 +63346164373435303066333439343564613030653962316139346437396638633035646164653061 +35663230323937326137316665303132663964383265636161356135613234326234626539383563 +35623564396465623534643063366563356662393934356532626264363734663361653062363564 +63333632393438636332656237323238323535306136396263343437653462633231666139626438 +33643431346464616562633231303463353831343137643936666630363336376130626562333036 +36336263306161626536633739336362343538643431386434623361626633313062363132373434 +62356439656139373462 diff --git a/files/tls-privkey.pem b/files/tls-privkey.pem index 58b444a..26a85a1 100644 --- a/files/tls-privkey.pem +++ b/files/tls-privkey.pem @@ -1,18 +1,90 @@ $ANSIBLE_VAULT;1.1;AES256 -61303034396434383432343035386165326236616263333131346565346337343566303039373062 -3138373432333164306365656461633665376165353263310a343966306335313433663538643836 -62306133656666653035346161353464653961653638353639616363326436336439656634663633 -3562373161333362370a313730616537393565323064383666396134343265353165643566393437 -66393063366166396662313437323937363237623262333531376565323563303839353961666337 -30393337643261366438336138386635303636643536633233336334626665646337613332646330 -31343466633932356430666563643962363465623839326563336139316364623266626132323062 -66323339663164306538313239646633636165363365616337363564383966616639303630636462 -64633330636463633265346330383566333234633630636566383030373061663366363131333861 -61376330326233666136393138366436303339343730633665346134613663303566333762396533 -31383138313030623636393562356133386463356262613766373536343235326334656362356237 -61646630343534633733613333613165356366383235656439356364636537656462373061653961 -35343932643037646464336232646466323833633838643337613933366466316636323665663633 -33313630633034303134663139666133613439643132636562616334303930396233323861623130 -35306665366162353537326335636361333138353063373532313964633337613666373465653035 -66643766373363353334383763323235656363643832633031316335306433626136393763386231 -3763 +62353962386130333935356530343936363639666564396466396161656466373434313066613165 +3739316238316538653739313732336435333566363538390a663830656161636335313830353535 +38373631656132366530353761356164656434383338363263343339666334323765366435623338 +3866646336376137390a333464343065333531363538303137323662653635636231663535336434 +32313939323963623763316238393861343735323361333536646436613333663237346163633061 +64313731383764346662313337663934653564626638343061633166653431353639666663656364 +36353835353339636665633732636434343635313466353036326136316465343739323431613235 +33663938333235353039656464343432306534653334346364303864336237383965636366663935 +61353834386263356439393831353339623361333733316366323933373464656236396164623064 +35323339313231373938383730376166303839396561346130366639663735323566326637366531 +31323265623134396436643461376539353037373564396635333937373139326537373637373765 +66333764383864633133363934306337323236323530343837316435373634356236386363643764 +37373736356435343036666464316164366335303736306265383466663938386166356133396132 +64383962633232316339356337643630666631626330636332646134653630346361663932306362 +31653739333761303337363939396234326336303737633334373437653534316633353964636430 +61663439323730376236626562303063333566336639313439353333336264636265303739623239 +65346265633862303430653036356338613132313039353266663531643637383434636338666334 +33653934366665326639313466333531363464346237303233643963303836616266363963306135 +35393664313963393239393662633534326434653764356436386162643661333865636533656563 +64323062353032626537336632376130346261316531663762663334353662346361636530363662 +64393064386238623635626433383361323132623833663364306138623964306639636163333333 +37306430396336653735383930316533633061646534316338353163613032333433656661386431 +35663633303130353239323539636435323731663062363066623464613635343665363633356162 +37633434613764623865653766346461393533393935333466363763373739346265396235383631 +36613733333236326165616536623662353862333937323064313530626133316333653137656132 +37326565653533363433356261326430663762373136383030323531633461616232383636626266 +30613564653931613164336166363137613433323136303135313532396166613065343462383636 +64346166643564366533636664656635323165653466326339623539323537653663623734343735 +38306232343934393234666636363039626562333530323833646137626431373465326166623136 +39373634653331646339393461636465663764616133323230326332333464616165333936376366 +62383762393332373163393162373030386339623535313733356338333038613731656238323461 +61376133653233356661323365613863623232646136303434656439653936653537626664393232 +36346663326438663736396362303738393763626539353138393763396665396637643732633434 +65326532653363613761373039306236626238373133313366653931383138623663343562613031 +31323238303333363161633938373239643862393261343531336166663364383166323236366136 +66666633393439383237323335623066326666646438626632376461376637613731383165386162 +63363336386236373363653335376235626637386433323337316234353531353365323039376236 +39366332306634336633353161313031323937386166626135323066333030336439383935353735 +39303730643465313062386562323165616630623230353639343432393361376239376231346164 +31653037646461383937643262393163653361363530303563353031623364383466636435646464 +37656663613961633034636430626532656361306137646633633730343130643266626661623732 +39393863323931303664613239356233623934313430633435613734656631336239653738393539 +39346261333232333136646236613130633163353837396534303332326636333664393063663236 +39373161613364613963616661383034363066393933643635336530353937623966666365313834 +61306532623664373261353231383734393132633637653765653733373039393361346535393162 +65326533396662623463383337303866343461353331316661316363393332636461643862373234 +63613764346330323965666236376264643466333163336238363439636531636438633763383662 +65636430336665323261663466613230333462373362393633366439393938383161653830666334 +35333031616236643830306336333064346666363433633265656166306338336263396630613134 +37633638313639656362306563346638613462353166326236303338623239393136313638326162 +32666163663864366561323939313033393635613431613531623436363566636564396439326334 +39323562393463333533663834323765333035376333366236363264366364396630663433653437 +35313030356363393763663333333461383830366136356263353864313936333137373264356164 +63303931326162376266346630616536663465356165643263643935663737616130616566333433 +34376665653234323935633334316662303731623135333634333765366333643932623033376630 +65323630366465353765333338333235653032333265393434376639333866336338663863353635 +38353932353961626562336365616532653534663737303234316666363737653233346537646266 +37646430383863616563366365626563643235313931323135666135323961336539333162633835 +38326539346130656535383438303830363133663636613631303635383666323038313930373435 +38633638663231373235306132346666626135303363313263613337646535643534666163646666 +37366538346332643266313731316131616339383534663062613564666263313238333439333434 +31666661353366393034336566643863323132323933623833653366316232383332343234393562 +30663237626561313234666164303035396339633666646632333261333534306663333265613335 +63623865356164643961376337346532343035353037623834363832656133343037646162303431 +37663234636433383736303730353130316532613430383536653364363337626433623463386233 +64653264663638313831646331323761393934643433306432363334323863396338336436393562 +61613962646430346637643061373136336234613263343938316361376561373664313835363037 +35353263373936313463633061303762323661386432636139346633333663623464636534383935 +31643931626130313431646138313966383834326137306662616439666136396434386665666461 +39383238623463313732656432663363653264346265346234336662643534633065333739336536 +36356439643332306532343836363065313937633735306238373539336134366434373533396639 +31353163653536303935643338616165623262356331323035393363363238653233376535353437 +33366137316638663939333363393437393133373139623131663531613937613232326564386639 +61366533616230656132613633613539356362353163363263383438366437616433363962323966 +33663335363832613835376662323230356235633437303338643161633937393035643130333038 +63623866653335396364373933376430386437373363333862653134313933356438626630663462 +35393962313965613064396566363937343465353963376362326665616637626561313136653666 +35646635663662643938393963376439326337646261323831393761613531333761663830383661 +38656330346262373961343166353161323630386633643138363735353861343962383165363665 +66613132396233666362356534636331383263616263353861336166316561396565333835633666 +64316135653137303264623030376265303361343832636432353663383365653561336633646339 +39303536666661376339623637396137323563373433653539353763646636363562353261393663 +66336264656134653866383232396330363334323332306530653432623333616163636264646634 +62626131623738333164643463636566663739323063303865613339343339643634623564336439 +65653034323738383630643438393164316333346638663466313565363336326433343864663638 +38303735626638653339333065623762666330656366306562613439383130393963373164303039 +63643737643035396535313335383261326337663439336630343830626639303635623231376636 +66313365636533643931396362663062303635303164653330356261356538623932326261656333 +36386239643539353365393136613535366530373537336261353030366162313838 From b2f800d25c9bedde23a9d85f8c6e837ce157fdbf Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Wed, 15 May 2024 16:05:25 -0400 Subject: [PATCH 22/31] Test webhook against newer Python We are running the server on F39, which has Python 3.12, so we should be testing there, instead of the older 3.9/3.10. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dbc848b..ce4018f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.11", "3.12"] steps: - uses: actions/checkout@v3 From f549f2244000cc1d5efe7e10429f013efef044ae Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 23 May 2024 03:54:17 -0400 Subject: [PATCH 23/31] Update Ansible playbooks for Fedora 39 Current Ansible / Jinja complains about the interpolation in a string, so move that to string concatenation instead. Grafana on Fedora 39 includes an SELinux configuration, which disables access to other ports, so toggle the boolean that allows connecting to Prometheus. --- README.md | 8 ++++---- create.yml | 2 +- matplotlib.org.yml | 6 ++++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1c9e883..7b4b00a 100644 --- a/README.md +++ b/README.md @@ -159,10 +159,10 @@ Note down the outputs to verify later, e.g., ``` # Use these for comparison when connecting yourself. -1024 SHA256:ExviVyBRoNKsZpgmIfBaejh1ElOpJ/9fC+ki2Fn5Xj4 root@venus.matplotlib.org (DSA) -256 SHA256:hLA7ePr0D4AgiC21IXowtbpcUNnTGgpPB7NOYepQtxg root@venus.matplotlib.org (ECDSA) -256 SHA256:MggFZQbZ7wID1Se2EmOwAm8AaJeA97L8sD8DhSrKy1g root@venus.matplotlib.org (ED25519) -3072 SHA256:MCkDgfbn0sMTCtvAtfD0HmGJV3LVTjpUj6IcfWRHRQo root@venus.matplotlib.org (RSA) +1024 SHA256:J2sbqvhI/VszBtVvPabgxyz6sRnGLrZUn0kqfv4doAM root@mercury.matplotlib.org (DSA) +256 SHA256:J0rOMayXhL1+5wbm4WQNpAvmscDjqwJjAtk1SLemRMI root@mercury.matplotlib.org (ECDSA) +256 SHA256:y8EDRGMpLWOW72x47MVKsAfSAl8JHjsOc/RGaiMTPGs root@mercury.matplotlib.org (ED25519) +3072 SHA256:AyuNO8FES5k9vobv0Pu9XpvtjVFZ1bTTNxb1lo+AuRA root@mercury.matplotlib.org (RSA) ``` Finally, you should reboot the droplet. This is due to a bug in cloud-init on diff --git a/create.yml b/create.yml index f23dbdd..ea0a501 100644 --- a/create.yml +++ b/create.yml @@ -69,7 +69,7 @@ # Must not be an existing name. - >- do_droplets.data | - selectattr('name', 'equalto', '{{ host }}.matplotlib.org') | + selectattr('name', 'equalto', host + '.matplotlib.org') | count == 0 # TODO: Also check that functional name doesn't already exist. - functional is regex('^web[0-9][0-9]$') diff --git a/matplotlib.org.yml b/matplotlib.org.yml index 428fc06..e7620d9 100644 --- a/matplotlib.org.yml +++ b/matplotlib.org.yml @@ -67,6 +67,12 @@ - podman state: present + - name: Allow Grafana to connect to Prometheus + ansible.posix.seboolean: + name: grafana_can_tcp_connect_prometheus_port + state: true + persistent: true + # Automatic updates # ################# - name: Enable automatic updates From 8193faece17e935f228dc40f6b3e6dfcf22d82fc Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Tue, 8 Oct 2024 21:13:45 -0400 Subject: [PATCH 24/31] Replace deprecated Caddyfile syntax This warns for every call to the `subproject` snippet. --- templates/Caddyfile.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index 37ba2d5..31cb3f5 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -3,10 +3,10 @@ (subproject) { # Caddy doesn't know this is a directory, so redirect to trailing / as # would usually happen for them. - redir /{args.0} /{args.0}/ permanent + redir /{args[0]} /{args[0]}/ permanent - handle_path /{args.0}/* { - root * {{ caddy.site_dir }}/{args.0} + handle_path /{args[0]}/* { + root * {{ caddy.site_dir }}/{args[0]} try_files {path}.html {path} file_server { hide .git From 86cd69623175f12df3e4a108e12cb3a9081fd240 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Tue, 8 Oct 2024 23:40:27 -0400 Subject: [PATCH 25/31] Move third-party packages to /thirdpartypackages Also, add an alias of `/3pp` and redirect the `/mpl-third-party` path to the new path. --- matplotlib.org.yml | 34 +++++++++++++++++----------------- templates/Caddyfile.j2 | 18 ++++++++++++++---- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/matplotlib.org.yml b/matplotlib.org.yml index e7620d9..7f00cdd 100644 --- a/matplotlib.org.yml +++ b/matplotlib.org.yml @@ -8,21 +8,21 @@ site_dir: "/usr/share/caddy" tls_config: "tls /etc/caddy/tls/cert.pem /etc/caddy/tls/privkey.pem" repos: - - mpl-brochure-site - - matplotlib.github.com - - basemap - - cheatsheets - - cycler - - devdocs - - governance - - ipympl - - matplotblog - - mpl-bench - - mpl-gui - - mpl-sphinx-theme - - mpl-third-party - - pytest-mpl - - data-prototype + mpl-brochure-site: + matplotlib.github.com: + basemap: + cheatsheets: + cycler: + devdocs: + governance: + ipympl: + matplotblog: + mpl-bench: + mpl-gui: + mpl-sphinx-theme: + mpl-third-party: thirdpartypackages + pytest-mpl: + data-prototype: vars_files: - files/webhook_vars.yml tasks: @@ -162,7 +162,7 @@ mode: 0755 owner: caddy group: caddy - loop: "{{ repos }}" + loop: "{{ repos.keys() }}" - name: Clone Git repositories become: true @@ -174,7 +174,7 @@ {{ (item == 'matplotlib.github.com') | ternary('main', 'gh-pages') }} - loop: "{{ repos }}" + loop: "{{ repos.keys() }}" # Caddy server setup # ################## diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index 31cb3f5..8becd7f 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -3,9 +3,9 @@ (subproject) { # Caddy doesn't know this is a directory, so redirect to trailing / as # would usually happen for them. - redir /{args[0]} /{args[0]}/ permanent + redir /{args[1]} /{args[1]}/ permanent - handle_path /{args[0]}/* { + handle_path /{args[1]}/* { root * {{ caddy.site_dir }}/{args[0]} try_files {path}.html {path} file_server { @@ -70,8 +70,8 @@ http://{{ caddy.addresses.main }}, http://{{ ansible_fqdn }} { root * {{ caddy.site_dir }} -{% for site in repos %} - import subproject {{ site }} +{% for site, path in repos.items() %} + import subproject {{ site }} {{ path | default(site, true) }} {% endfor %} # Hide mpl-altair until the site is fixed. @@ -96,6 +96,16 @@ http://{{ caddy.addresses.main }}, http://{{ ansible_fqdn }} { } } + # Redirect the repo-named third-party packages path to the more-generic path. + redir /3pp /thirdpartypackages/ # And also add a shortcut. + @mpl-third-party path /mpl-third-party /mpl-third-party/* + handle @mpl-third-party { + route { + uri strip_prefix /mpl-third-party + redir * /thirdpartypackages{uri} + } + } + # Place the brochure site at the top level. @brochure file { root {{ caddy.site_dir }}/mpl-brochure-site From 08a0ec72f6e1c2c2adc96ba23a6b986cf917f348 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Tue, 8 Oct 2024 23:46:09 -0400 Subject: [PATCH 26/31] Redirect old top-level files to versioned docs --- matplotlib.org.yml | 2 ++ templates/Caddyfile.j2 | 64 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/matplotlib.org.yml b/matplotlib.org.yml index 7f00cdd..dee35ab 100644 --- a/matplotlib.org.yml +++ b/matplotlib.org.yml @@ -8,6 +8,8 @@ site_dir: "/usr/share/caddy" tls_config: "tls /etc/caddy/tls/cert.pem /etc/caddy/tls/privkey.pem" repos: + # Key is repo name, value is the site /path/ (which defaults to the repo + # name if not set). mpl-brochure-site: matplotlib.github.com: basemap: diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index 8becd7f..fcb7624 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -114,6 +114,70 @@ http://{{ caddy.addresses.main }}, http://{{ ansible_fqdn }} { rewrite / /mpl-brochure-site/index.html rewrite @brochure /mpl-brochure-site{http.matchers.file.relative} + # Redirect any of the old top-level files to the versioned docs. + redir /citing.html /stable/project/citing.html permanent + redir /contents.html /stable/users/ permanent + redir /downloads.html /stable/users/installing/ permanent + redir /gallery.html /stable/gallery/ permanent + redir /py-modindex.html /stable/py-modindex.html permanent +{# 3.5.0 was the last version to be synced to the top-level directory. + We also try for stable, just to go to the best version possible. #} +{%- with old_versions = [ + "stable", + "3.5.0", + "3.4.3", "3.4.2", "3.4.1", "3.4.0", + "3.3.4", "3.3.3", "3.3.2", "3.3.1", "3.3.0", + "3.2.2", "3.2.1", "3.2.0", + "3.1.3", "3.1.1", "3.1.0", + "3.0.3", "3.0.2", "3.0.0", + "2.2.5", "2.2.4", "2.2.3", "2.2.2", "2.2.0", + "2.1.2", "2.1.1", "2.1.0", + "2.0.2", "2.0.1", "2.0.0", + "1.5.3", "1.5.1", "1.5.0", + "1.4.3", "1.4.2", "1.4.1", "1.4.0", + "1.3.1", "1.3.0", "1.2.1", +] %} +{%- with old_toplevel_dirs = [ + "_downloads", + "_images", + "_modules", + "_panels_static", + "_sources", + "_static", + "api", + "devel", + "examples", + "faq", + "gallery", + "glossary", + "mpl_examples", + "mpl_toolkits", + "plot_directive", + "plot_types", + "pyplots", + "resources", + "tutorials", + "users", +] %} + @old-toplevel-dirs { + path{% for dir in old_toplevel_dirs %} /{{dir}} /{{dir}}/{% endfor +%} + file { + root {{ caddy.site_dir }}/matplotlib.github.com + try_files{% for version in old_versions %} /{{version}}/{path}/index.html{% endfor +%} + } + } + redir @old-toplevel-dirs {http.matchers.file.relative} permanent + @old-toplevel-dir-contents { + path{% for dir in old_toplevel_dirs %} /{{dir}}/*{% endfor +%} + file { + root {{ caddy.site_dir }}/matplotlib.github.com + try_files{% for version in old_versions %} /{{version}}/{path}.html /{{version}}/{path}{% endfor +%} + } + } + redir @old-toplevel-dir-contents {http.matchers.file.relative} permanent +{%- endwith -%} +{%- endwith +%} + # Finally try any of the versioned docs. handle { root * {{ caddy.site_dir }}/matplotlib.github.com From 76760f1d2bbb711d65fe6f2c63f6b14a7921fb02 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 13 Feb 2025 05:24:05 -0500 Subject: [PATCH 27/31] Improve logging from webhook When an error occurs in the webhook (either with itself or due to invalid input), it returns a 4xx/5xx and some error text. For security reasons, Caddy reduces all of these results to a simple 400/503. Since aiohttp only logs the response code, and Caddy only logs admin problems, this information was lost. We don't really do anything with access logs, so just enable them only for the errors. --- templates/Caddyfile.j2 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index fcb7624..08a4e59 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -22,6 +22,15 @@ https://{{ caddy.addresses.webhook }} { http://{{ caddy.addresses.webhook }} { {% endif %} + # Include access logs when an error occurs, since we mask any internal errors + # from escaping to the outside world, but otherwise don't log. + log { + output discard + } + log errors { + no_hostname + } + root * {{ caddy.site_dir }} # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#delivery-headers @@ -41,6 +50,9 @@ http://{{ caddy.addresses.webhook }} { # Don't leak out internal problems. @error status 4xx 5xx handle_response @error { + log_name errors + log_append api_error_code {rp.status_code} + log_append api_error_text {rp.status_text} error 400 } } @@ -51,6 +63,9 @@ http://{{ caddy.addresses.webhook }} { # Don't leak out internal problems. @error status 4xx 5xx handle_response @error { + log_name errors + log_append api_error_code {rp.status_code} + log_append api_error_text {rp.status_text} error 503 } } From f55633611c5720a3de00cb1ff77fc06bdbd92a07 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 13 Feb 2025 05:52:20 -0500 Subject: [PATCH 28/31] Remove manual setting of firewall during droplet creation The firewall is now attached to droplet tags, and thus is automatically added to new droplets since we tag them. --- README.md | 2 +- create.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 7b4b00a..15e4d12 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ Initial setup The summary of the initial setup is: 1. Create the droplet with monitoring and relevant SSH keys. -2. Assign new droplet to the matplotlib.org project and the Web firewall. +2. Assign new droplet to the matplotlib.org project. 3. Add DNS entries pointing to the server on CloudFlare. 4. Grab the SSH host fingerprints. 5. Reboot. diff --git a/create.yml b/create.yml index ea0a501..938f687 100644 --- a/create.yml +++ b/create.yml @@ -91,8 +91,6 @@ community.digitalocean.digital_ocean_droplet: state: present name: "{{ host }}.matplotlib.org" - firewall: - - Web image: fedora-39-x64 monitoring: true project: matplotlib.org From 749216a66bc98d91d01c978d91d5602a366f170d Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 13 Feb 2025 05:55:01 -0500 Subject: [PATCH 29/31] Document what permissions the tokens need --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 15e4d12..12e8129 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,14 @@ Before you can run our Ansible playbooks, you need to meet the following prerequisites: * Create a DigitalOcean API token, and pass it to the inventory generator by - setting the `DO_API_TOKEN` environment variable. + setting the `DO_API_TOKEN` environment variable. The API token must have + access to the following scopes: + - Read: droplet, firewall, monitoring, project, ssh_key + - Create: droplet + - Update: droplet, monitoring, project * If you are creating a new droplet, and want to configure DNS as well, then - create a CloudFlare API token, and pass it to the Ansible playbook by setting - the `CLOUDFLARE_TOKEN` environment variable. + create a CloudFlare API token with DNS edit permissions, and pass it to the + Ansible playbook by setting the `CLOUDFLARE_TOKEN` environment variable. * Set the vault decryption password of the Ansible vaulted file with our secrets. This may be done by setting the `ANSIBLE_VAULT_PASSWORD_FILE` environment variable to point to a file containing the password. From 7f8359fb9fb232bf32fde69263b441df0b6a9aa8 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 13 Feb 2025 21:38:24 -0500 Subject: [PATCH 30/31] Add more information to droplet creation Also, set tags in CloudFlare DNS. --- README.md | 9 ++++++++- collections/requirements.yml | 2 +- create.yml | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 12e8129..8dd21fa 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,8 @@ ansible-playbook create.yml --extra-vars "host=pluto functional=web99 ssh_keys=' The playbook will create the server, as well as add DNS records on CloudFlare. Note, you must set `DO_API_TOKEN` and `CLOUDFLARE_TOKEN` in the environment to -access these services. +access these services. The droplet ID and IP address will be printed at the +end of the playbook. Then, to ensure you are connecting to the expected server, you should grab the SSH host keys via the DigitalOcean Droplet Console: @@ -173,6 +174,12 @@ Finally, you should reboot the droplet. This is due to a bug in cloud-init on DigitalOcean, which generates a new machine ID after startup, causing system logs to be seem invisible. +This can be done from the Console, or via the CLI: + +``` +doctl compute droplet-action reboot +``` + Running Ansible --------------- diff --git a/collections/requirements.yml b/collections/requirements.yml index 4a795f3..712d59b 100644 --- a/collections/requirements.yml +++ b/collections/requirements.yml @@ -2,5 +2,5 @@ collections: - name: ansible.posix - name: community.general - version: ">=2.0.0" + version: ">=8.0.0" - name: community.digitalocean diff --git a/create.yml b/create.yml index 938f687..f927d60 100644 --- a/create.yml +++ b/create.yml @@ -115,6 +115,8 @@ map(attribute='ip_address') | first }} + tags: + - website zone: matplotlib.org - name: Setup functional DNS for droplet on CloudFlare @@ -124,8 +126,23 @@ record: "{{ functional }}" type: CNAME value: "{{ host }}.matplotlib.org" + tags: + - website zone: matplotlib.org + - name: Print droplet info + ansible.builtin.debug: + msg: + - "Droplet ID is {{ new_droplet.data.droplet.id }}" + - >- + First Public IPv4 is {{ + (new_droplet.data.droplet.networks.v4 | selectattr('type', 'equalto', 'public')).0.ip_address | + default('', true) }} + - >- + First Private IPv4 is {{ + (new_droplet.data.droplet.networks.v4 | selectattr('type', 'equalto', 'private')).0.ip_address | + default('', true) }} + vars: # We currently name servers based on planets in the Solar System. valid_planets: From 7f56982d4727f342a2d23aebfe30de981fcdd07c Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 13 Feb 2025 22:42:12 -0500 Subject: [PATCH 31/31] Update to Fedora 41 --- README.md | 7 +++---- create.yml | 2 +- files/dnf5-automatic.conf | 2 ++ matplotlib.org.yml | 11 ++++++++--- 4 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 files/dnf5-automatic.conf diff --git a/README.md b/README.md index 8dd21fa..98400bf 100644 --- a/README.md +++ b/README.md @@ -164,10 +164,9 @@ Note down the outputs to verify later, e.g., ``` # Use these for comparison when connecting yourself. -1024 SHA256:J2sbqvhI/VszBtVvPabgxyz6sRnGLrZUn0kqfv4doAM root@mercury.matplotlib.org (DSA) -256 SHA256:J0rOMayXhL1+5wbm4WQNpAvmscDjqwJjAtk1SLemRMI root@mercury.matplotlib.org (ECDSA) -256 SHA256:y8EDRGMpLWOW72x47MVKsAfSAl8JHjsOc/RGaiMTPGs root@mercury.matplotlib.org (ED25519) -3072 SHA256:AyuNO8FES5k9vobv0Pu9XpvtjVFZ1bTTNxb1lo+AuRA root@mercury.matplotlib.org (RSA) +256 SHA256:p6MiA8+IO1WcpXHDOQ4rhiVCo+MDxWB7ehfNfxvbDkU root@venus.matplotlib.org (ECDSA) +256 SHA256:RfDahJqnQFLeFN+zl9f+hmB+W05OoZK26NfNQkj6KtY root@venus.matplotlib.org (ED25519) +3072 SHA256:tYwdULlz5/XP5Ze7PCj9XpO3VIMEZkiOiFuhr9nke34 root@venus.matplotlib.org (RSA) ``` Finally, you should reboot the droplet. This is due to a bug in cloud-init on diff --git a/create.yml b/create.yml index f927d60..47a4a0a 100644 --- a/create.yml +++ b/create.yml @@ -91,7 +91,7 @@ community.digitalocean.digital_ocean_droplet: state: present name: "{{ host }}.matplotlib.org" - image: fedora-39-x64 + image: fedora-41-x64 monitoring: true project: matplotlib.org region: tor1 diff --git a/files/dnf5-automatic.conf b/files/dnf5-automatic.conf new file mode 100644 index 0000000..5d40e60 --- /dev/null +++ b/files/dnf5-automatic.conf @@ -0,0 +1,2 @@ +[commands] +apply_updates = yes diff --git a/matplotlib.org.yml b/matplotlib.org.yml index dee35ab..f9be944 100644 --- a/matplotlib.org.yml +++ b/matplotlib.org.yml @@ -45,7 +45,7 @@ - name: Install server maintenance ansible.builtin.dnf: name: - - dnf-automatic + - dnf5-plugin-automatic - fail2ban state: present @@ -63,8 +63,8 @@ name: - golang-github-prometheus - golang-github-prometheus-alertmanager - - golang-github-prometheus-node-exporter - grafana + - node-exporter # Remove this when Loki is packaged. - podman state: present @@ -77,9 +77,14 @@ # Automatic updates # ################# + - name: Configure automatic updates + ansible.builtin.copy: + src: dnf5-automatic.conf + dest: /etc/dnf/dnf5-plugins/automatic.conf + - name: Enable automatic updates ansible.builtin.systemd: - name: dnf-automatic-install.timer + name: dnf5-automatic.timer enabled: true state: started