From f84a5a75e4010c4ec530dd7fe544684b4a5b2192 Mon Sep 17 00:00:00 2001
From: Daniel Ecer <de-code@users.noreply.github.com>
Date: Tue, 28 Jun 2022 11:48:12 +0100
Subject: [PATCH 1/2] use compressed ingress log related files to reduce disk
 space usage (#7)

* compress ingress log before upload to reduce bandwidth

* introduced CLOUDWATCH_JSONL_GZ_FILE to reduce hardcoding

* moved gzip compression down

* generate schema from compressed ingress log

* generate gzipped ingress jsonl file

* compress intermediate ingress jsonl files

* avoid uncompressing cloudwatch logs

* introduced input variable for clarity

* introduced target file variable for more clarity

* added logging of intermediate connversions

* using shell parameter expansion for readability
---
 Makefile                                      | 21 ++++++++++---------
 ...nvert-cloudwatch-logs-to-bigquery-jsonl.sh |  8 -------
 scripts/convert-cloudwatch-logs-to-jsonl.sh   | 18 ----------------
 ...ed-cloudwatch-logs-to-bigquery-jsonl-gz.sh | 15 +++++++++++++
 ...ert-gzipped-cloudwatch-logs-to-jsonl-gz.sh | 20 ++++++++++++++++++
 scripts/download-from-cloudwatch.sh           |  1 -
 6 files changed, 46 insertions(+), 37 deletions(-)
 delete mode 100755 scripts/convert-cloudwatch-logs-to-bigquery-jsonl.sh
 delete mode 100755 scripts/convert-cloudwatch-logs-to-jsonl.sh
 create mode 100755 scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh
 create mode 100755 scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh

diff --git a/Makefile b/Makefile
index ff545f4..f0b009e 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,7 @@ CLOUDWATCH_FROM_DATE = $(shell ./scripts/determine-cloudwatch-from-date-based-on
 CLOUDWATCH_TO_DATE = $(shell date '+%Y-%m-%d')
 CLOUDWATCH_TARGET_DIR = ./logs/cloudwatch
 CLOUDWATCH_JSONL_FILE = ./logs/ingress.jsonl
+CLOUDWATCH_JSONL_GZ_FILE = $(CLOUDWATCH_JSONL_FILE).gz
 CLOUDWATCH_JSONL_SCHEMA_FILE = $(CLOUDWATCH_JSONL_FILE).bq-schema.json
 
 
@@ -84,17 +85,17 @@ download-events-from-s3:
 		"$(CLOUDWATCH_TO_DATE)" \
 		"$(CLOUDWATCH_TARGET_DIR)"
 
-.convert-cloudwatch-logs-to-jsonl:
-	./scripts/convert-cloudwatch-logs-to-jsonl.sh \
+.convert-gzipped-cloudwatch-logs-to-jsonl-gz:
+	./scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh \
 		"$(CLOUDWATCH_TARGET_DIR)" \
-		"$(CLOUDWATCH_JSONL_FILE)"
+		"$(CLOUDWATCH_JSONL_GZ_FILE)"
 
-.generate-schema-for-cloudwatch-jsonl-file: venv
-	cat "$(CLOUDWATCH_JSONL_FILE)" \
+.generate-schema-for-cloudwatch-jsonl-gz-file: venv
+	zcat "$(CLOUDWATCH_JSONL_GZ_FILE)" \
 		| venv/bin/generate-schema \
 		> "$(CLOUDWATCH_JSONL_SCHEMA_FILE)"
 
-.upload-ingress-jsonl-to-bigquery:
+.upload-ingress-jsonl-gz-to-bigquery:
 	bq load \
 		--project_id=elife-data-pipeline \
 		--noreplace \
@@ -102,14 +103,14 @@ download-events-from-s3:
 		--schema_update_option=ALLOW_FIELD_ADDITION \
 		--source_format=NEWLINE_DELIMITED_JSON \
 		de_proto.sciety_ingress_v1 \
-		"$(CLOUDWATCH_JSONL_FILE)"
+		"$(CLOUDWATCH_JSONL_GZ_FILE)"
 
 .do-upload-ingress-logs-from-cloudwatch-to-bigquery:
 	$(MAKE) .cloudwatch-show-info
 	$(MAKE) .export-and-download-from-cloudwatch
-	$(MAKE) .convert-cloudwatch-logs-to-jsonl
-	$(MAKE) .generate-schema-for-cloudwatch-jsonl-file
-	$(MAKE) .upload-ingress-jsonl-to-bigquery
+	$(MAKE) .convert-gzipped-cloudwatch-logs-to-jsonl-gz
+	$(MAKE) .generate-schema-for-cloudwatch-jsonl-gz-file
+	$(MAKE) .upload-ingress-jsonl-gz-to-bigquery
 
 .upload-ingress-logs-from-cloudwatch-to-bigquery:
 	@if [ "$(CLOUDWATCH_FROM_DATE)" = "$(CLOUDWATCH_TO_DATE)" ]; then \
diff --git a/scripts/convert-cloudwatch-logs-to-bigquery-jsonl.sh b/scripts/convert-cloudwatch-logs-to-bigquery-jsonl.sh
deleted file mode 100755
index 9f52ac2..0000000
--- a/scripts/convert-cloudwatch-logs-to-bigquery-jsonl.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-cat $1 \
-    | sed -e 's/[^ ]* //' \
-    | jq --compact-output 'del(.kubernetes) | del(.docker)' \
-    > $1.jsonl
diff --git a/scripts/convert-cloudwatch-logs-to-jsonl.sh b/scripts/convert-cloudwatch-logs-to-jsonl.sh
deleted file mode 100755
index daced73..0000000
--- a/scripts/convert-cloudwatch-logs-to-jsonl.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-local_cloudwatch_dir="$1"
-target_jsonl_file="$2"
-
-if [ -z "${local_cloudwatch_dir}" ] || [ -z "${target_jsonl_file}" ]; then
-    echo "Usage: $0 <local cloudwatch dir> <target jsonl file>"
-    exit 1
-fi
-
-echo "converting ${local_cloudwatch_dir} to ${target_jsonl_file}"
-
-find "${local_cloudwatch_dir}" -type 'f' \
-    | grep -v jsonl \
-    | xargs -n 1 ./scripts/convert-cloudwatch-logs-to-bigquery-jsonl.sh
-(find "${local_cloudwatch_dir}" -type 'f' | grep jsonl | xargs cat) > "${target_jsonl_file}"
diff --git a/scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh b/scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh
new file mode 100755
index 0000000..a4b66b3
--- /dev/null
+++ b/scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euo pipefail
+
+log_gz_file="$1"
+common_filename=${log_gz_file%.*}
+target_jsonl_gz_file="$common_filename.jsonl.gz"
+
+echo "converting $log_gz_file to $target_jsonl_gz_file"
+
+zcat $log_gz_file \
+    | sed -e 's/[^ ]* //' \
+    | jq --compact-output 'del(.kubernetes) | del(.docker)' \
+    | gzip - \
+    > $target_jsonl_gz_file
diff --git a/scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh b/scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh
new file mode 100755
index 0000000..b34b39c
--- /dev/null
+++ b/scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+local_cloudwatch_dir="$1"
+target_jsonl_gz_file="$2"
+
+if [ -z "${local_cloudwatch_dir}" ] || [ -z "${target_jsonl_gz_file}" ]; then
+    echo "Usage: $0 <local cloudwatch dir> <target jsonl gz file>"
+    exit 1
+fi
+
+echo "converting ${local_cloudwatch_dir} to ${target_jsonl_gz_file}"
+
+find "${local_cloudwatch_dir}" -type 'f' \
+    | grep -v jsonl \
+    | xargs -n 1 ./scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh
+
+echo "combining jsonl files to ${target_jsonl_gz_file}"
+(find "${local_cloudwatch_dir}" -type 'f' | grep jsonl.gz | xargs zcat | gzip -) > "${target_jsonl_gz_file}"
diff --git a/scripts/download-from-cloudwatch.sh b/scripts/download-from-cloudwatch.sh
index 87f2f7b..c63bd7a 100755
--- a/scripts/download-from-cloudwatch.sh
+++ b/scripts/download-from-cloudwatch.sh
@@ -14,4 +14,3 @@ echo "downloading from ${logs_url} to ${target_dir}"
 
 mkdir -p "${target_dir}"
 aws s3 cp --recursive ${logs_url} "${target_dir}"
-gunzip -r "${target_dir}"

From bacd7d7a65a4a31ca8bf01ed0618b94f6247180c Mon Sep 17 00:00:00 2001
From: Daniel Ecer <de-code@users.noreply.github.com>
Date: Wed, 6 Jul 2022 14:29:41 +0100
Subject: [PATCH 2/2] fixed ingress upload issue on mac caused by zcat adding
 .Z

---
 Makefile                                                       | 3 ++-
 .../convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh    | 3 ++-
 scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh         | 2 +-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index f0b009e..e68e574 100644
--- a/Makefile
+++ b/Makefile
@@ -91,7 +91,8 @@ download-events-from-s3:
 		"$(CLOUDWATCH_JSONL_GZ_FILE)"
 
 .generate-schema-for-cloudwatch-jsonl-gz-file: venv
-	zcat "$(CLOUDWATCH_JSONL_GZ_FILE)" \
+	cat "$(CLOUDWATCH_JSONL_GZ_FILE)" \
+		| zcat \
 		| venv/bin/generate-schema \
 		> "$(CLOUDWATCH_JSONL_SCHEMA_FILE)"
 
diff --git a/scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh b/scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh
index a4b66b3..c62a7d1 100755
--- a/scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh
+++ b/scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh
@@ -8,7 +8,8 @@ target_jsonl_gz_file="$common_filename.jsonl.gz"
 
 echo "converting $log_gz_file to $target_jsonl_gz_file"
 
-zcat $log_gz_file \
+cat $log_gz_file \
+    | zcat \
     | sed -e 's/[^ ]* //' \
     | jq --compact-output 'del(.kubernetes) | del(.docker)' \
     | gzip - \
diff --git a/scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh b/scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh
index b34b39c..515b0d1 100755
--- a/scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh
+++ b/scripts/convert-gzipped-cloudwatch-logs-to-jsonl-gz.sh
@@ -17,4 +17,4 @@ find "${local_cloudwatch_dir}" -type 'f' \
     | xargs -n 1 ./scripts/convert-gzipped-cloudwatch-logs-to-bigquery-jsonl-gz.sh
 
 echo "combining jsonl files to ${target_jsonl_gz_file}"
-(find "${local_cloudwatch_dir}" -type 'f' | grep jsonl.gz | xargs zcat | gzip -) > "${target_jsonl_gz_file}"
+(find "${local_cloudwatch_dir}" -type 'f' | grep jsonl.gz | xargs gunzip -c | gzip -) > "${target_jsonl_gz_file}"