build(Makefile): work on start targets; custom minio and memcached Dockerfiles, and more

.gitignore

@@ -1,5 +1,6 @@
 .Dockerfile
 dist*
+develop
 node_modules/
 assets/icons
 assets/favicons

Makefile

@@ -2,21 +2,30 @@ SHELL=bash
 # MAKE=make -d
 
 export CONTAINER_COMMAND ?= podman
-export CONTAINER_RUN ?= $(CONTAINER_COMMAND) run -it --network=host
+export CONTAINER_RUN ?= $(CONTAINER_COMMAND) run -it -d --network=host
 
 export IN_CONTAINER ?= false
 export IN_CI ?= false
 
+# TODO: deprecated
 export WATCH
 export db ?= -cf
 
+export CONTAINER_ID
+export FRADRIVE_SERVICE
+
+# TODO: to be deprecated
 export DEV_PORT_HTTP
 export DEV_PORT_HTTPS
 
 export IMAGE_REGISTRY = docker.io
 export MEMCACHED_IMAGE = $(IMAGE_REGISTRY)/memcached:latest
 export MINIO_IMAGE = $(IMAGE_REGISTRY)/minio/minio:latest
-export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest
+export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest # TODO: needs different port than 1025 to avoid conflicts
+
+export DEVELOP
+export CONTAINER_FILE
+export CONTAINER_FILE_CONTENT
 
 # HELP HEADER START
 # To see the definition of all available targets, take a look into the Makefile.
@@ -31,8 +40,6 @@ export MAILDEV_IMAGE = $(IMAGE_REGISTRY)/maildev/maildev:latest
 ###########################
 ##### GENERAL TARGETS #####
 
-.PHONY: --%
-
 .PHONY: help
 # HELP: print out this help message
 help:
@@ -56,7 +63,7 @@ clean-all: clean
 	$(CONTAINER_COMMAND) volume prune --force
 
 .PHONY: release
-# HELP: unfinished
+# HELP: create, commit and push a new release
 release:
 	./.gitlab-ci/version.pl -changelog CHANGELOG.md
 	git add CHANGELOG.md
@@ -65,12 +72,16 @@ release:
 	git commit -m "chore(release): $${VERSION}"
 	git push
 
+# TODO: rewrite shell targets to enter shell inside currently running containers, do not just launch new containers!
 .PHONY: %-shell
-%-shell:
+# HELP: launches bash-shell inside backend/frontend container
+%-shell: ENTRYPOINT=/bin/bash
+%-shell: --%-shell;
+--%-shell:
 	MOUNT_DIR=/mnt/fradrive ; \
 	FRADRIVE_SERVICE=$* ; \
 	$(MAKE) -- --image-build FRADRIVE_SERVICE=$${FRADRIVE_SERVICE} ; \
-	$(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --entrypoint /bin/bash --name fradrive.$${FRADRIVE_SERVICE}.interactive.$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$${FRADRIVE_SERVICE}
+	$(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env CONTAINER_FILE=$(CONTAINER_FILE) --entrypoint $(ENTRYPOINT) --name fradrive.$${FRADRIVE_SERVICE}.interactive.$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$${FRADRIVE_SERVICE}
 
 ##### GENERAL TARGETS #####
 ###########################
@@ -86,25 +97,19 @@ release:
 #   - additionally provide targets for termination of running containers
 .PHONY: serve
 # HELP: serve frontend, backend, and database
-serve:
+serve: serve-database serve-memcached serve-minio serve-frontend serve-backend;
-	$(MAKE) serve-database &
-	$(MAKE) serve-memcached &
-	$(MAKE) serve-minio &
-#	$(MAKE) serve-maildev & # TODO: needs different port than 1025 to avoid conflicts
-	$(MAKE) serve-frontend &
-	$(MAKE) serve-backend
 
 .PHONY: compile
 # HELP: compile frontend and backend
-compile: compile-frontend compile-backend
+compile: compile-frontend compile-backend;
 
 .PHONY: lint
 # HELP: lint frontend and backend
-lint: lint-frontend lint-backend
+lint: lint-frontend lint-backend;
 
 .PHONY: test
 # HELP: test frontend, backend, and check internationalization
-test: test-frontend test-backend i18n-check
+test: test-frontend test-backend i18n-check;
 
 ##### UNIFIED FRONTEND/BACKEND TARGETS #####
 ############################################
@@ -120,8 +125,7 @@ test: test-frontend test-backend i18n-check
 --%-frontend: --containerized-static-frontend --containerized-well-known-frontend;
 
 # HELP(compile-frontend): compile frontend
---compile-frontend: static well-known
+--compile-frontend: static well-known;
-	npm run build
 
 .PHONY: serve-frontend
 # HELP: serve frontend (watch file changes)
@@ -181,16 +185,21 @@ well-known: static;
 --%-prod-backend: stackopts=--flag uniworx:-dev
 --%-prod-backend: --image-build --containerized-%-backend;
 
+# TODO: to be deprecated by start-%
 .PHONY: serve-backend
 # HELP: serve backend
 serve-backend:
-	DEV_PORT_HTTP=`docker/backend/dev_port.pl 3000 | tee .dev-port-http`; \
+	DEV_PORT_HTTP=`utils/next_free_port.pl 3000 | tee .dev-port-http`; \
-	DEV_PORT_HTTPS=`docker/backend/dev_port.pl 3443 | tee .dev-port-https`; \
+	DEV_PORT_HTTPS=`utils/next_free_port.pl 3443 | tee .dev-port-https`; \
 	$(MAKE) -- --containerized---serve-dev-backend DEV_PORT_HTTP=$${DEV_PORT_HTTP} DEV_PORT_HTTPS=$${DEV_PORT_HTTPS}
+#	DEV_PORT_HTTP=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \
+#	DEV_PORT_HTTPS=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \
+#	DEVELOP=$(MAKE) develop/
 --serve-dev-backend: start.sh
 	DEV_PORT_HTTP=`cat .dev-port-http`; \
 	DEV_PORT_HTTPS=`cat .dev-port-https`; \
 	./start.sh
+#	./utils/watchrun.sh develop/ ./start.sh
 
 # HELP(compile-backend): compile backend
 --compile-backend:
@@ -206,7 +215,6 @@ serve-backend:
 
 # HELP(db-cf-backend): clear and fill database. requires running postgres
 # TODO (db-m-$MIGRATION-backend): apply migration (see src/Model/Migration/Definition.hs for list of available migrations)
-PHONY: --db-%-backend
 --db-%-dev-backend: .stack
 	SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true} ; \
 	AVSPASS=${AVSPASS:-nopasswordset} ; \
@@ -225,6 +233,7 @@ PHONY: --db-%-backend
 ############################
 ##### DATABASE TARGETS #####
 
+# TODO: to be deprecated by start-%
 .PHONY: serve-database
 # HELP: serve database
 serve-database: --containerized-database;
@@ -236,14 +245,6 @@ serve-database: --containerized-database;
 	  $(CONTAINER_RUN) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$(FRADRIVE_SERVICE) ; \
 	fi
 
-.PHONY: serve-memcached
-serve-memcached:
-	$(CONTAINER_RUN) $(MEMCACHED_IMAGE)
-
-.PHONY: serve-minio
-serve-minio:
-	$(CONTAINER_RUN) $(MINIO_IMAGE) -- server $$(mktemp)
-
 ##### DATABASE TARGETS #####
 ############################
 
@@ -251,7 +252,6 @@ serve-minio:
 #############################
 ##### CONTAINER TARGETS #####
 
-.PHONY: --containerized-%-frontend
 --containerized-%-frontend: FRADRIVE_SERVICE=frontend
 --containerized-%-frontend: --image-build
 	$(MAKE) -- --image-run-$*-frontend
@@ -260,6 +260,14 @@ serve-minio:
 --containerized-%-backend: --image-build
 	$(MAKE) -- --image-run-$*-backend 
 
+--containerized-%-minio: FRADRIVE_SERVICE=minio
+--containerized-%-minio: --image-build
+	$(MAKE) -- --image-run-$*-minio
+
+--containerized-%-memcached: FRADRIVE_SERVICE=memcached
+--containerized-%-memcached: --image-build
+	$(MAKE) -- --image-run-$*-memcached
+
 .PHONY: image-rebuild
 # HELP: rebuild the stated docker image (frontend, backend, database instead of %)
 image-rebuild-%:
@@ -281,13 +289,127 @@ image-rebuild-%:
 	if [ "$(IN_CONTAINER)" == "true" ] ; then \
 	  $(MAKE) -- $* ; \
 	else \
-	  $(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env WATCH=$(WATCH) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') localhost/fradrive/$(FRADRIVE_SERVICE) ; \
+	  CONTAINER_ID=`$(CONTAINER_RUN) -v $(PWD):$${MOUNT_DIR} --env IN_CONTAINER=true --env FRADRIVE_MAKE_TARGET=$* --env CONTAINER_FILE=$(CONTAINER_FILE) --env CONTAINER_FILE_CONTENT=$(CONTAINER_FILE_CONTENT) --env WATCH=$(WATCH) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') localhost/fradrive/$(FRADRIVE_SERVICE)` ; \
+	  echo "CONTAINER_ID=$${CONTAINER_ID}" >> $(CONTAINER_FILE); \
 	fi
 
+# .PHONY: new-backend
+# new-backend:
+# 	LAST_DATE=`ls -1 develop | tail -n1`
+# 	if [[ -z "$${LAST_DATE}" || -e `ls develop/$${LAST_DATE}/database` ]]; then \
+#	  mkdir -p develop/$$(date +'%Y-%m-%dT%H-%M-%S') \
+#	fi
+#	serve-backend
+
+#new-database:
+#	mkdir -p develop/$$(date +'%Y-%m-%dT%H-%M-%S')
+#	$(MAKE) serve-database
+
+# TODO: work in progress
+# - compute used ports and provide as env vars
+# - start container in detached mode
+# - save container id with ports as content as file
+# .PHONY: serve-%
+# serve-backend:
+#	DEV_PORT_HTTP=`docker/backend/dev_port.pl 3000 | tee develop/$${DATE}/fradrive.backend.$${DATE}`; \
+#	DEV_PORT_HTTPS=`docker/backend/dev_port.pl 3443 | tee .dev-port-https`; \
+#	$(MAKE) -- --containerized---start-dev-backend DEV_PORT_HTTP=$${DEV_PORT_HTTP} DEV_PORT_HTTPS=$${DEV_PORT_HTTPS}
+#--serve-dev-backend: start.sh
+#	DEV_PORT_HTTP=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \
+#	DEV_PORT_HTTPS=`grep 'https:' develop/$(DATE)/backend/$(CONTAINER_ID) | sed 's/.*://'`; \
+#	./utils/watchrun.sh develop/ ./start.sh
+
+
+#CURRENT_DEVELOP = $(shell mkdir -p develop && (ls -1 develop | tail -n1))
+CURRENT_DEVELOP = $(shell if [[ ! -e develop ]] ; then mkdir develop; $(MAKE) develop ; fi && (ls -1 develop | tail -n1))
+DEVELOP      = develop/$(CURRENT_DEVELOP)
+DATE    := $(shell date +'%Y-%m-%dT%H-%M-%S')
+SET_DEVELOP = $(eval DEVELOP=develop/$$(DATE))
+
+.PHONY: develop
+develop:
+	$(SET_DEVELOP)
+	mkdir -p develop/$(DATE)
+--ensure-develop:
+	if ! [[ -e develop ]]; then \
+	  $(MAKE) develop; \
+	fi
+
+start: develop start-database start-memcached start-minio start-frontend start-backend;
+
+start-database:  BASE_PORTS = "PGPORT=5432"
+start-database:  SINGLETON = true
+start-memcached: BASE_PORTS = "MEMCACHED_PORT=11211"
+start-memcached: SINGLETON = true
+start-minio:     BASE_PORTS = "UPLOAD_S3_PORT=9000"
+start-minio:     SINGLETON = true
+
+start-backend:   BASE_PORTS = "DEV_PORT_HTTP=3000" "DEV_PORT_HTTPS=3443"
+start-backend:   SINGLETON = false
+
+start-frontend:  SINGLETON = false
+
+start-%: FRADRIVE_SERVICE = %
+start-%: PORTS = $(foreach PORT,$(BASE_PORTS),$(shell utils/next_free_port.pl $(PORT)))
+start-%: --ensure-develop
+	echo "$*"
+	echo "$(DATE)"
+	echo "DEV $(DEVELOP)"
+	if [[ "$(SINGLETON)" = "true" ]]; then \
+	  CONTAINER_FILE=$(DEVELOP)/$* ; \
+	else \
+	  DEVDIR=$(DEVELOP)/$*; \
+	  mkdir -p $${DEVDIR}; \
+	  I=`ls $${DEVDIR} | sort -n | tail -n1`; \
+	  echo "$${I}"; \
+	  CONTAINER_FILE=$${DEVDIR}/$$(($${I}+1)); \
+	fi ; \
+	echo "$(PORTS)" | sed 's/ /\n/g' > $${CONTAINER_FILE} ; \
+	$(MAKE) -- --containerized---start-$* CONTAINER_FILE=$${CONTAINER_FILE}
+
+--start-backend: start.sh
+	DEV_PORT_HTTP=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTP=' | sed 's/DEV_PORT_HTTP=//'`; \
+	DEV_PORT_HTTPS=`cat $(CONTAINER_FILE) | grep 'DEV_PORT_HTTPS=' | sed 's/DEV_PORT_HTTPS=//'`; \
+	./utils/watchrun-container.sh "$(CONTAINER_FILE)" ./start.sh
+
+--containerized---start-database: FRADRIVE_SERVICE=database
+# port forwarding is disabled in --network=host mode; nevertheless it is stated here for documentation reasons
+--containerized---start-database: docker/database/initdb.sh docker/database/pg_hba.conf docker/database/postgresql.conf docker/database/schema.sql --image-build
+	if [ "$(IN_CONTAINER)" == "false" ] ; then \
+	  $(CONTAINER_RUN) --name fradrive.$(FRADRIVE_SERVICE).$$(date +'%Y-%m-%dT%H-%M-%S') fradrive/$(FRADRIVE_SERVICE) ; \
+	fi
+
+# TODO: mkdir inside develop/$(DATE)/minio instead of mktemp
+# TODO: remove dir in stop-minio
+#--containerized---start-minio:
+#	MINIO_DIR=`mktemp` ; \
+#	echo "MINIO_DIR=$${MINIO_DIR}" > $(DEVELOP)/minio ; \
+#	CONTAINER_ID=`$(CONTAINER_RUN) -v $(PWD):/mnt/fradrive --file docker/minio/Dockerfile -- (server $${MINIO_DIR})` ; \
+#	echo "CONTAINER_ID=$${CONTAINER_ID}" >> $(DEVELOP)/minio
+
+# TODO: work in progress
+# - kill container and remove file from .develop
+# TODO: find way to stop a single container, e.g. by removing the file
+.PHONY: stop-%
+stop-%: --stop-%
+	$(MAKE) stop-container CONTAINER_FILE=TODO
+.PHONY: --stop-minio
+--stop-minio:
+	MINIO_DIR=`cat $(DEVELOP)/minio`
+	rm -rf $${MINIO_DIR}
+
+stop-container-by-id:
+	podman stop $(CONTAINER_ID)
+stop-container-by-file:
+	CONTAINER_ID=`grep 'CONTAINER_ID=' $(CONTAINER_FILE) | sed 's/CONTAINER_ID=//'` ; \
+	$(MAKE) stop-container-by-id CONTAINER_ID=$${CONTAINER_ID}
+
 ##### CONTAINER TARGETS #####
 #############################
 
 
+# TODO: move targets below to better location
+
 .PHONY: i18n-check
 # HELP: check internationalization
 i18n-check: --image-run---i18n-check
@@ -298,3 +420,5 @@ i18n-check: --image-run---i18n-check
 %.lock:
 	[ -e $@ ] || touch $@
 	flock -en $@ true
+
+.PHONY: --%

docker/backend/Dockerfile

@@ -23,7 +23,7 @@ RUN make .stack STACK_ROOT=${STACK_ROOT} IN_CONTAINER=true
 RUN stack build yesod-bin
 
 ENV FRADRIVE_MAKE_TARGET=serve-backend
-ENTRYPOINT make -- ${FRADRIVE_MAKE_TARGET} STACK_ROOT="${STACK_ROOT}" IN_CONTAINER=true
+ENTRYPOINT cat ${CONTAINER_FILE} > /container-file && make -- ${FRADRIVE_MAKE_TARGET} STACK_ROOT="${STACK_ROOT}" IN_CONTAINER=true CONTAINER_FILE="${CONTAINER_FILE}"
 
 EXPOSE 3000/tcp
 EXPOSE 3443/tcp

docker/backend/start.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -e
+
+# SPDX-FileCopyrightText: 2022-2024 Sarah Vaupel <sarah.vaupel@uniworx.de>, Gregor Kleen <gregor.kleen@ifi.lmu.de>, Sarah Vaupel <vaupel.sarah@campus.lmu.de>, Steffen Jost <jost@tcs.ifi.lmu.de>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+
+__HOST=${HOST:-$(hostname -s | awk '{ print $0; }')}
+
+export DETAILED_LOGGING=${DETAILED_LOGGING:-true}
+export LOG_ALL=${LOG_ALL:-false}
+export LOGLEVEL=${LOGLEVEL:-info}
+export DUMMY_LOGIN=${DUMMY_LOGIN:-true}
+export SERVER_SESSION_ACID_FALLBACK=${SERVER_SESSION_ACID_FALLBACK:-true}
+export SERVER_SESSION_COOKIES_SECURE=${SERVER_SESSION_COOKIES_SECURE:-false}
+export COOKIES_SECURE=${COOKIES_SECURE:-false}
+export ALLOW_DEPRECATED=${ALLOW_DEPRECATED:-true}
+export ENCRYPT_ERRORS=${ENCRYPT_ERRORS:-false}
+export RIBBON=${RIBBON:-${__HOST:-localhost}}
+export APPROOT=${APPROOT:-http://localhost:$DEV_PORT_HTTP}
+export AVSPASS=${AVSPASS:-nopasswordset}
+unset HOST
+
+# export PORT_OFFSET=$(((16#$(sha256sum <<<"$(hostname -f):''${basePath}" | head -c 16)) % 1000))
+
+# move-back() {
+#     mv -vT .stack-work .stack-work-run
+#     [[ -d .stack-work-build ]] && mv -vT .stack-work-build .stack-work
+# }
+
+# if [[ -d .stack-work-run ]]; then
+#     [[ -d .stack-work ]] && mv -vT .stack-work .stack-work-build
+#     mv -vT .stack-work-run .stack-work
+#     trap move-back EXIT
+# fi
+
+if ! [ -z "$(which yesod)" ]
+then
+  yesod devel -p "$DEV_PORT_HTTP" -q "$DEV_PORT_HTTPS" $@
+elif ! [ -z "$(which stack)" ]
+then
+  stack exec -- yesod devel -p "$DEV_PORT_HTTP" -q "$DEV_PORT_HTTPS" $@
+else
+  exit 1
+fi

docker/memcached/Dockerfile

@@ -0,0 +1,4 @@
+FROM docker.io/memcached:latest
+
+USER root
+ENTRYPOINT /mnt/fradrive/utils/watchrun.sh "/mnt/fradrive/${CONTAINER_FILE}" "su memcache -c /usr/local/bin/docker-entrypoint.sh"

docker/minio/Dockerfile

@@ -0,0 +1,6 @@
+FROM docker.io/minio/minio:latest
+
+ENV MINIO_DIR = $(mktemp)
+
+WORKDIR /mnt/fradrive
+ENTRYPOINT ./utils/watchrun.sh "${CONTAINER_FILE}" "/usr/bin/docker-entrypoint.sh server ${MINIO_DIR}" "rm -rf ${MINIO_DIR}"

utils/next_free_port.pl

@@ -3,7 +3,9 @@
 use strict;
 use warnings;
 
-my $port = $ARGV[0];
+my $portDef = $ARGV[0];
+$portDef=~m/^(\D*)(\d+)(.*)/ or die "$0: We need a start port as parameter but got none!\n";
+my ($pre, $port, $post) = ($1, $2, $3);
 my $used_ports = `netstat -tulan`;
 
 my %p=();
@@ -16,4 +18,4 @@ for(split m/\R/, $used_ports) {
 
 $port++ while $p{$port};
 
-print $port
+print "$pre$port$post"

utils/runwrapper.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo $$ > /tmp/pidwrapper
+
+$1

utils/watchrun-container.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+STOPSCRIPT="make stop-container-by-id CONTAINFER_FILE=${CONTAINER_FILE} ; $3"
+
+./utils/watchrun.sh "$1" "$2"

utils/watchrun.sh

@@ -6,13 +6,14 @@ STOPSCRIPT=$3
 
 touch "$FILENAME"
 
-$STARTSCRIPT &
+(
-
 while [ -e "$FILENAME" ] ; do
   inotifywait -e ATTRIB -t 10 "$FILENAME" > /dev/null 2&>/dev/null
 done
 
-kill %1
-
 $STOPSCRIPT
 
+kill `cat /tmp/pidwrapper`
+) &
+
+/mnt/fradrive/utils/runwrapper.sh "$STARTSCRIPT"