Fastly: configure logs streaming for dl.k8s.io

Switch from BigQuery to Datadog for logs streaming

Signed-off-by: Arnaud Meukam <ameukam@gmail.com>

Author: ameukam

infra/fastly/terraform/dl.k8s.io/data.tf

@@ -14,6 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+data "google_secret_manager_secret_version_access" "datadog_api_key" {
+  secret  = "datadog_fastly_logs_streaming"
+  project = "k8s-infra-releases-prod"
+}
+
 data "google_secret_manager_secret_version_access" "gcs_reader_access_key" {
   secret  = "fastly_reader_sa_access_key"
   project = "k8s-infra-releases-prod"

infra/fastly/terraform/dl.k8s.io/fastly-log-format.tftpl

@@ -0,0 +1 @@
+{ "ddsource": "fastly", "ddtags": "app:${dd_app},env:${dd_env}", "service": "${service_name}", "date": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_start": "%%{begin:%Y-%m-%dT%H:%M:%S%Z}t", "time_end": "%%{end:%Y-%m-%dT%H:%M:%S%Z}t", "http": { "request_time_ms": %D, "method": "%m", "url": "%%{json.escape(req.url)}V", "useragent": "%%{User-Agent}i", "referer": "%%{Referer}i", "protocol": "%H", "request_x_forwarded_for": "%%{X-Forwarded-For}i", "status_code": "%s" }, "network": { "client": { "ip": "%h", "name": "%%{client.as.name}V", "number": "%%{client.as.number}V", "connection_speed": "%%{client.geo.conn_speed}V" }, "destination": { "ip": "%A" }, "geoip": { "geo_city": "%%{client.geo.city.utf8}V", "geo_country_code": "%%{client.geo.country_code}V", "geo_continent_code": "%%{client.geo.continent_code}V", "geo_region": "%%{client.geo.region}V" }, "bytes_written": %B, "bytes_read": %%{req.body_bytes_read}V }, "host": "%%{if(req.http.Fastly-Orig-Host, req.http.Fastly-Orig-Host, req.http.Host)}V", "origin_host": "%v", "is_ipv6": %%{if(req.is_ipv6, "true", "false")}V, "is_tls": %%{if(req.is_ssl, "true", "false")}V, "tls_client_protocol": "%%{json.escape(tls.client.protocol)}V", "tls_client_servername": "%%{json.escape(tls.client.servername)}V", "tls_client_cipher": "%%{json.escape(tls.client.cipher)}V", "tls_client_cipher_sha": "%%{json.escape(tls.client.ciphers_sha)}V", "tls_client_tlsexts_sha": "%%{json.escape(tls.client.tlsexts_sha)}V", "is_h2": %%{if(fastly_info.is_h2, "true", "false")}V, "is_h2_push": %%{if(fastly_info.h2.is_push, "true", "false")}V, "h2_stream_id": "%%{fastly_info.h2.stream_id}V", "request_accept_content": "%%{Accept}i", "request_accept_language": "%%{Accept-Language}i", "request_accept_encoding": "%%{Accept-Encoding}i", "request_accept_charset": "%%{Accept-Charset}i", "request_connection": "%%{Connection}i", "request_dnt": "%%{DNT}i", "request_forwarded": "%%{Forwarded}i", "request_via": "%%{Via}i", "request_cache_control": "%%{Cache-Control}i", "request_x_requested_with": "%%{X-Requested-With}i", "request_x_att_device_id": "%%{X-ATT-Device-Id}i", "content_type": "%%{Content-Type}o", "is_cacheable": %%{if(fastly_info.state~"^(HIT|MISS)$", "true","false")}V, "response_age": "%%{Age}o", "response_cache_control": "%%{Cache-Control}o", "response_expires": "%%{Expires}o", "response_last_modified": "%%{Last-Modified}o", "response_tsv": "%%{TSV}o", "server_datacenter": "%%{server.datacenter}V", "req_header_size": %%{req.header_bytes_read}V, "resp_header_size": %%{resp.header_bytes_written}V, "socket_cwnd": %%{client.socket.cwnd}V, "socket_nexthop": "%%{client.socket.nexthop}V", "socket_tcpi_rcv_mss": %%{client.socket.tcpi_rcv_mss}V, "socket_tcpi_snd_mss": %%{client.socket.tcpi_snd_mss}V, "socket_tcpi_rtt": %%{client.socket.tcpi_rtt}V, "socket_tcpi_rttvar": %%{client.socket.tcpi_rttvar}V, "socket_tcpi_rcv_rtt": %%{client.socket.tcpi_rcv_rtt}V, "socket_tcpi_rcv_space": %%{client.socket.tcpi_rcv_space}V, "socket_tcpi_last_data_sent": %%{client.socket.tcpi_last_data_sent}V, "socket_tcpi_total_retrans": %%{client.socket.tcpi_total_retrans}V, "socket_tcpi_delta_retrans": %%{client.socket.tcpi_delta_retrans}V, "socket_ploss": %%{client.socket.ploss}V }

infra/fastly/terraform/dl.k8s.io/services.tf

@@ -65,15 +65,15 @@ resource "fastly_service_vcl" "files" {
     window         = 4
   } */
 
-  logging_bigquery {
-    dataset      = "fastly_bigquery_cdn_dl_k8s_io"
-    project_id   = "k8s-infra-public-pii"
-    name         = "BigQuery logging"
-    table        = "fastly_bigquery_cdn_dl_k8s_io_logs"
-    account_name = "fastly-bigquery-logging-sa"
-    email        = "fastly-logging@datalog-bulleit-9e86.iam.gserviceaccount.com"
-
-    format = "%%{strftime(\\{\"%Y-%m-%dT%H:%M:%S%z\"\\}, time.start)}V|%%{client.as.number}V|%%{if(fastly.ff.visits_this_service == 0, \"true\", \"false\")}V"
+  logging_datadog {
+    name   = "dd-oss-k8s"
+    token  = data.google_secret_manager_secret_version_access.datadog_api_key.secret_data
+    region = "US5"
+    format = templatefile("${path.module}/fastly-log-format.tftpl", {
+      service_name = "cdn.dl.k8s.io",
+      dd_app       = "releases",
+      dd_env       = "prod",
+    })
   }
 
   snippet {