avec md des differents bouts de scripts

Specifications.md

@@ -19,7 +19,6 @@ deploy_service(service)
 {  
    - determines if the service is on this server  
     if it is:   
-    - checks the environment (/data/mounted and new nginx conf exists)  
     - creates directories (secret, data, http) if necessary   
     - calls the bash scripts specific to the service if they exist (deploy_as)  
     - creates user and .env variables if necessary  
@@ -41,15 +40,19 @@ remove_service(service) //should there be some kind of less absolute removal?
 **main.cpp** 
 - reads the instructions
 If deploy all:
-    - tests system requirements (/data/mounted)  
+    - checks the environment (/data/mounted and new nginx conf exists)  
+    - prepares directories
     - prepares Nginx  
     - prepares Docker  
     - prepares .env   
     for each service:  
         - deploy_service(service) 
+    - (re)start nginx 
     - cleans  
 else if deploy one service: 
+    - checks the environment (/data/mounted and new nginx conf exists)  
     - deploy_service(service)  
+    - (re)start nginx
 else if remove all:  
     for each service:   
         - remove_service(service)  
@@ -65,12 +68,10 @@ else if remove one service:
 - nginx
     preparation
     deployment
-        deployment
+        writing new conf
-        testing
-        cleaning
-    restart
-        restart
         testing
+        replacing old conf
+        start/restart
         cleaning       
     stop
 - wireguard

docker.md

@@ -0,0 +1,35 @@
+
+docker_service="$(echo "$service" | tr '.' '_')"
+
+## deploy 
+
+
+	if $deploy ; then
+    	section "Logging to registry"
+    	# XXX Login to docker registry
+
+    	section "Pulling images"
+    	docker-compose pull
+		if [ "$?" -ne 0 ] ; then
+			echo "PULL FAILED"
+		fi
+
+    	section "Starting service"
+    	run docker-compose up -d --remove-orphans
+		[ "$?" -ne 0 ] && echo "Erreur docker compose" && returncode=1
+	else
+		section "Removing containers"
+    	run docker-compose down --rmi all --remove-orphans
+	fi
+fi
+
+## clean
+
+if ! "$deploy" ; then
+	section "Remove stray containers"
+	while read container ; do
+		[ -z "$container" ] && continue || true
+		echo "Removing $container"
+		run docker rm "$container"
+	done <<< "$(docker ps | grep "$docker_service" | cut -d ' ' -f 1)"
+fi

letsencrypt.md

@@ -0,0 +1,84 @@
+
+
+. /etc/jeancloud.env
+
+
+# Variable
+acmeroot=/var/www/letsencrypt
+
+# Création du répertoire
+mkdir -p "$acmeroot"
+
+
+for file in "$nginx_conf_path"* ; do
+  if $verbose ; then
+    echo '-------------------------'
+    echo "$file"
+  fi
+
+  service_name="$(basename "$file")"
+
+  if [ -d "$dns_certs_path/$service_name" ] ; then
+	  echo "$service_name is handled by dnscerts"
+	  continue
+  fi
+    
+  # Getting just the domain names
+  domains="$(extract_domain_nginx_conf.sh "$file")"
+  (grep '^[[:blank:]]*[^#][[:blank:]]*server_name' "$file" | sed 's/ _ / /g' | sed   's/server_name//g' | sed 's/default_server//g' | sed -e 's/^[[:space:]]*//' -e 's/;$//'  | sort -u)
+  if [ -n "$domains" ] ; then
+    # If using dummy cert, disabling it
+	dummy_cert.sh "$service_name" remove
+
+    echo "$domains"
+    # adding -d before every domain
+    domains="-d $(echo $domains | sed 's/ / -d /g')"
+
+    # Run certbot
+    command="certbot certonly -n --expand --agree-tos --webroot -w "$acmeroot" --email contact@jean-cloud.org --cert-name "$(basename $file)" $domains"
+    if $verbose ; then
+      echo $command
+    fi
+    out="$($command 2>&1)"
+    result="$?"
+    
+    if [ "$result" -eq 0 ] && [[ "$out" = *"Certificate not yet due for renewal; no action taken."* ]]; then
+      echo "Cert still valid"
+    elif [ "$result" -eq 0 ] ; then
+      echo "Cert renewed or obtained"
+      #new_cert="$(echo "$out" | grep -oE '/etc/letsencrypt/live/.*/fullchain.pem')"
+      #echo "'$new_cert'"
+      #new_cert_dir="$(dirname "$out")"
+      #echo "'$new_cert_dir'"
+    
+      #if [ -d "$new_cert_dir" ] ; then
+      #  echo "New cert dir : '$new_cert_dir'"
+      #  echo "cp '$new_cert_dir/*' '/data/proxy/certs/'"
+      #else
+      #  echo "Error parsiong dir name"
+      #fi
+    
+    elif [ "$result" -eq 1 ] ; then
+      echo "Cert failed"
+      echo "     ------------------------------------------"
+      echo "$out"
+      echo "     ------------------------------------------"
+    else
+        echo "Unknown error : $result.\n$out"
+    fi
+  fi
+done
+
+ls /etc/letsencrypt/live/*000* &> /dev/null
+if [ "$?" -eq 0 ] ; then
+    echo " ---------------------------------------------------------------------------------------------"
+    echo "Bad certs detected in letsencrypt dir. Nginx conf wont work…"
+    echo "rm -r /etc/letsencrypt/live/*000* /etc/letsencrypt/archive/*000* /etc/letsencrypt/renewal/*000*"
+    echo " ---------------------------------------------------------------------------------------------"
+fi
+
+
+# nginx
+call nginx test
+call nginx reload
+depuis le main ou ici ? 

main_deploy_service.md

@@ -0,0 +1,78 @@
+
+## checks if service on server 
+
+failed=""
+
+while IFS=';' read -r id username service target
+do
+
+	echo -n "$service -> "
+    [ ! -d "/docker/$service" ] && die "/docker/$service directory not found"
+
+	# Check if service target is localhost
+	[[ "$(getent hosts $target)" != "::1 "* ]] && echo 'Not here' && continue
+	
+	echo "Deploying"
+    deploy_service.sh "$service" "noreload"
+
+	if [ "$?" -ne 0 ] ; then
+		failed="$failed $service"
+	fi
+
+done < <(grep -v '^#' /docker/services.csv)   
+
+
+## user id
+uid="$(($services_uid_start + $id))"
+
+## environment variables
+
+[ -f .env ] && set -a && . .env && set +a
+[ -f "$SECRET_DIR/.env" ] && set -a && . "$SECRET_DIR/.env" && set +a
+
+## useful directories 
+
+if "$deploy" ; then
+	run mkdir -p "$DATA_DIR" "$HTTP_DIR"
+	run chown $uid "$DATA_DIR"
+	run chmod 751 "$DATA_DIR"
+	run chown $uid:www-data -R "$HTTP_DIR"
+	if [ -d "$SECRET_DIR" ] ; then
+		run chown $uid "$SECRET_DIR" -R
+		run chmod 751 "$SECRET_DIR" -R
+	fi
+else
+    [ -d "$HTTP_DIR" ] && rm -r "$HTTP_DIR"
+fi
+
+## run bash scripts
+
+returncode=0
+if "$deploy" ; then
+	if [ -x deploy.sh ] ; then
+		run ./deploy.sh
+		[ "$?" -ne 0 ] && echo "Erreur deploy.sh" && returncode=1
+	fi
+	if [ -x deploy_user.sh ] ; then
+		deploy_as "$service"
+		[ "$?" -ne 0 ] && echo "Erreur deploy_user.sh" && returncode=1
+	fi
+else
+	[ -x undeploy.sh ] && run ./undeploy.sh
+fi
+
+
+## docker 
+
+if [ -f "/docker/$service/docker-compose.yml" ] && [ -n "$(grep  '^[^#]*services' "/docker/$service/docker-compose.yml")" ] ; then
+// launch docker deploy
+
+## nginx
+
+if [ -f "/docker/$service/nginx_server.conf" ] ; then
+//launch nginx deploy 
+
+## wireguard
+
+if [ -f  "/docker/$service" -name "wg-*.sh" ]
+//launch wg deploy

main_preparation_deploy_all.md

@@ -0,0 +1,57 @@
+## Check system requirements
+
+[ ! -f /data/mounted ] && die "/data is not mounted"  
+
+if [ ! -d "$new_nginx_conf_path" ] ; then
+	die "Can’t deploy service in degraded state. $new_nginx_conf_path dir is missing, please run deployall.sh first"
+fi
+
+## prepare directories 
+mkdir -p /docker /data  
+run chown root:root /docker  
+run chown root:root /data  
+run chmod 755 /docker  
+run chmod 755 /data  
+
+## prepare environment variables 
+
+JC_ENV=/etc/jeancloud.env  
+
+dns_certs_path=/data/dnscerts.jean-cloud.org/certs/live  
+http_certs_path=/etc/letsencrypt/live  
+proxy_dir=/etc/nginx  
+ 
+cat > "$JC_ENV" <<EOF  
+proxy_dir='$proxy_dir'  
+nginx_conf_path='$proxy_dir/sites-enabled/'  
+new_nginx_conf_path='$proxy_dir/new-sites-enabled'  
+dns_certs_path='$dns_certs_path'  
+http_certs_path='$http_certs_path'  
+dummy_cert_path='$http_certs_path/dummy'  
+servicefile=/docker/services.csv
+services_uid_start=2000  
+EOF  
+	
+while IFS=';' read -r id username service server      
+do
+	dir="/docker/$service"
+    [ ! -d "$dir" ] && continue    
+
+	line_in_file "HTTP_DIR='/srv/http/$service'" "$dir/.env"  
+	line_in_file "DATA_DIR='/data/$service'" "$dir/.env"  
+	line_in_file "SECRET_DIR='/data/secrets/$service'" "$dir/.env"  
+	line_in_file "DOCKER_DIR='$dir'" "$dir/.env"     
+	line_in_file "JC_SERVICE='$service'" "$dir/.env"  
+	#line_in_file "HOME='/data/$service'" "$dir/.env"  
+	line_in_file "NET='172.29.$id'" "$dir/.env"  
+	line_in_file "USER='$username'" "$dir/.env"  
+	line_in_file "JC_ID='$id'" "$dir/.env"  
+
+	cert="$(findcert.sh "$service")" || true  
+	if [ -n "$cert" ] ; then  
+		line_in_file "JC_CERT='$cert'" "$dir/.env"  
+	fi  
+		 
+done < <(grep -v '^#' /docker/services.csv)      
+
+

nginx_library.md

@@ -0,0 +1,68 @@
+## NGINX library
+preparation
+deployment
+    deployment
+    testing
+    cleaning
+restart
+    restart
+    testing
+    cleaning        
+stop
+
+## Preparation 
+
+driglibash_section_prefix="[Prepare nginx] "
+section "Delete new conf directory (to start from scratch)"
+run rm -rf "$new_nginx_conf_path"
+
+section "Create new conf file (for tests purposes)"
+sed "s#$nginx_conf_path#$new_nginx_conf_path/#" "/docker/_proxy/nginx.conf" > "$proxy_dir/new_nginx.conf"
+
+section "Create proxy dir"
+run mkdir -p "$proxy_dir" 
+
+section "Create new conf directory"
+run mkdir -p "$new_nginx_conf_path"
+
+## deploy
+### write new conf
+if [ -f "/docker/$service/nginx_server.conf" ] ; then
+    section "Copy nginx conf"
+    run cp "/docker/$service/nginx_server.conf" "$new_nginx_conf_path/$service"
+
+    section "Template nginx conf with vars from '.env' file"
+    run template.sh "/docker/$service/.env" < "/docker/$service/nginx_server.conf" > "$new_nginx_conf_path/
+    (template : 	bash -c 'set -a && . '"$1"' &&  envsubst "$(cat '"$1"' | grep -o ^.*= | sed "s/=//" | sed "s/^/$/")"')
+    $service"
+fi
+
+### testing 
+section "Testing nginx conf"
+run nginx -t -c /etc/nginx/new_nginx.conf
+[ "$?" -ne 0 ] && echo "Erreur nginx" && returncode=1
+
+### replace old conf
+
+section "Update nginx conf"
+run rm -rf "$nginx_conf_path"
+run cp -r "$new_nginx_conf_path" "$nginx_conf_path"
+run cp "/docker/_proxy/nginx.conf" "$proxy_dir/nginx.conf"
+
+### sane testing
+section "Test nginx conf to be sure"
+run nginx -t
+
+## restart or start
+if [ -z "$(cat /var/run/nginx.pid)" ] ; then
+    section "Start nginx"
+    run nginx
+else
+    section "Reload nginx"
+    run nginx -s reload
+fi
+
+## cleaning
+section "Cleaning"
+rmdir "$DATA_DIR" "$HTTP_DIR" 2>/dev/null || true
+

wireguard.md

@@ -0,0 +1,77 @@
+# deploy
+### If there is a wireguard vpn script
+for file in $( find "/docker/$service" -name "wgns-*.sh") ; do
+	section "Managing wg interface $(basename "$file")"
+    if [ -x "$file" ] ; then
+        wgif="$(basename "$file")"
+        wgif="${wgif:5:-3}"
+		if [ -z "$wgif" ] ; then
+			echo "No wireguard name for $file"
+			returncode=1
+			continue
+		fi
+        "$file" $wgif > "/etc/wireguard/$wgif.conf"
+		if "$deploy" ; then
+        	#run systemctl enable "wg-quick@$wgif"
+        	run managewg.sh start "$wgif"
+			[ "$?" -ne 0 ] && echo "Erreur wireguard" && returncode=1
+		else
+        	run managewg.sh stop "$wgif"
+		fi
+    fi
+done
+
+### If there is a wireguard vpn template
+for file in $( find "/docker/$service" -name "wg-*.sh") ; do
+	section "Creating wg iface $(basename "$file")"
+    if [ -x "$file" ] ; then
+        wgif="$(basename "$file")"
+        wgif="${wgif:3:-3}"
+		if [ -z "$wgif" ] ; then
+			echo "No wireguard name for $file"
+			returncode=1
+			continue
+		fi
+		#run template.sh "/docker/$service/.env" < "$file" > "/etc/wireguard/$wgif.conf"
+        "$file" $wgif > "/etc/wireguard/$wgif.conf"
+
+		if "$deploy" ; then
+        	run systemctl enable "wg-quick@$wgif"
+        	run startwg.sh start "$wgif"
+			[ "$?" -ne 0 ] && echo "Erreur wireguard" && returncode=1
+		else
+        	run managewg.sh stop "$wgif"
+		fi
+    fi
+done
+
+## start
+run="ip netns exec $wgif"
+
+start () {
+	echo "Starting $wgif"
+
+	# Create netns if needed
+	if ! ip netns | grep -q "$wgif" ; then
+		ip netns add "$wgif"
+	fi
+	
+	# Create iface
+	if ! ip link | grep -q "$wgif" ; then
+		ip link add "$wgif" type wireguard
+		ip link set "$wgif" netns "$wgif"
+	fi
+
+	#$run wg-quick up "$wgif"
+	$run wg setconf "$wgif" "/etc/wireguard/$wgif.conf"
+}
+
+stop () {
+	echo "Stoping $wgif"
+	$run wg-quick down "$wgif" || true
+}
+
+reload () {
+	echo "Reloading $wgif"
+	$run wg syncconf "$wgif" <(wg-quick strip "$wgif")
+}