chore(docker): no root & fill

flake.nix

@@ -90,6 +90,7 @@
             "^(assets|app|hlint|load|messages|models|src|templates|test|testdata|wflint)(/.*)?$"
             "^config(/(archive-types|mimetypes|personalised-sheet-files-collate|settings\.yml|submission-blacklist|test-settings\.yml|video-types|wordlist\.txt))?$"
             "^routes$"
+            "^testdata(/.*)?$"
           ];
 
           pkgs = import nixpkgs {
@@ -97,11 +98,12 @@
             config.allowUnfree = true;
           };
           overlays = [
-            # remove once https://github.com/NixOS/nix/pull/5266 has landed in https://hub.docker.com/r/nixpkgs/nix-unstable
+            # remove once https://github.com/NixOS/nix/pull/5266 has landed in nixpkgs used here (see flake.lock)
             (import ./nix/aws-patch.nix)
-            (import "${docker-nixpkgs}/overlay.nix")
             # end remove
 
+            (import "${docker-nixpkgs}/overlay.nix")
+
             (import ./nix/maildev)
             haskell-nix.overlay
             (import ./nix/uniworx { inherit inputs frontendSource backendSource; })

nix/docker/default.nix

@@ -20,8 +20,8 @@ let
 
     contents = with final; [
       uniworx.uniworx.components.exes.uniworx
-      prev.dockerTools.binSh
+      prev.dockerTools.binSh findutils coreutils
-    ] ++ optionals isDemo [ postgresql_12 memcached ];
+    ] ++ optionals isDemo [ postgresql_12 memcached uniworx.uniworx.components.exes.uniworxdb ];
 
     runAsRoot = ''
       #!${final.stdenv.shell}
@@ -38,21 +38,15 @@ let
       install -d -g uniworx -o uniworx -m 0755 /var/log/uniworx
 
       ${optionalString isDemo ''
-          groupadd -r postgres
+          install -d -g uniworx -o uniworx -m 0750 /var/lib/postgres
-          useradd -r -g postgres -d /var/lib/postgres -M postgres
-          install -d -g postgres -o postgres -m 0750 /var/lib/postgres
 
-          groupadd -r memcached
+          install -d -g uniworx -o uniworx -m 0750 /var/lib/memcached
-          useradd -r -g memcached -d /var/lib/memcached -M memcached
-          install -d -g memcached -o memcached -m 0750 /var/lib/memcached
 
-          gpasswd -a uniworx postgres
+          install -d -g uniworx -o uniworx -m 0755 /var/log/postgres
-
+          install -d -g uniworx -o uniworx -m 0755 /var/log/memcached
-          install -d -g postgres -o postgres -m 0755 /var/log/postgres
-          install -d -g memcached -o memcached -m 0755 /var/log/memcached
 
           mkdir -p /run
-          install -d -g postgres -o postgres -m 0755 /run/postgres
+          install -d -g uniworx -o uniworx -m 0755 /run/postgres
       ''}
     '';
 
@@ -60,7 +54,6 @@ let
       let
         entrypoint = prev.writeScriptBin "uniworx-entrypoint" ''
           #!${final.zsh}/bin/zsh -xe
-          export PATH=${final.su}/bin:${final.findutils}/bin:${final.coreutils}/bin:/bin
 
           cTime=$(date -Is)
 
@@ -76,11 +69,11 @@ let
                 pgNew=1
               fi
 
-              [[ -z "''${pgNew}" ]] || su postgres -c "initdb --no-locale --encoding=UTF8 -D ''${pgDir}"
+              [[ -z "''${pgNew}" ]] || initdb --no-locale --encoding=UTF8 --username postgres --pgdata ''${pgDir}
-              su postgres -c "pg_ctl start -D ''${pgDir} -l ''${pgLogFile} -w -o '-k ''${pgSockDir} -c listen_addresses= -c hba_file=${postgresHba} -c unix_socket_permissions=0777 -c max_connections=9990 -c shared_preload_libraries=pg_stat_statements -c auto_explain.log_min_duration=100ms'"
+              pg_ctl start -D ''${pgDir} -l ''${pgLogFile} -w -o "-k ''${pgSockDir} -c listen_addresses= -c hba_file=${postgresHba} -c unix_socket_permissions=0777 -c max_connections=9990 -c shared_preload_libraries=pg_stat_statements -c auto_explain.log_min_duration=100ms"
               [[ -z "''${pgNew}" ]] || psql -f ${postgresSchema} postgres postgres
 
-              su memcached -c "cd /var/lib/memcached; memcached -p 11212" &>/var/log/memcached/''${cTime}.log &
+              ( cd /var/lib/memcached; memcached -p 11212 ) &>/var/log/memcached/''${cTime}.log &
               export SESSION_MEMCACHED_HOST=localhost
               export SESSION_MEMCACHED_PORT=11212
           ''}
@@ -95,7 +88,11 @@ let
             done < <(find "''${configDir}" \( -name '*.yml' -o -name '*.yaml' \) -print0 | sort -rz)
           fi
           configs+=('${uniworxConfig}')
-          exec -- su uniworx -c "cd /var/lib/uniworx; uniworx ''${configs}"
+          cd /var/lib/uniworx
+          ${optionalString isDemo ''
+              [[ -z "''${pgNew}" ]] || uniworxdb -f ''${configs}
+          ''}
+          exec -- uniworx ''${configs}
         '';
         postgresSchema = prev.writeText "schema.sql" ''
           CREATE USER uniworx WITH SUPERUSER;
@@ -112,6 +109,7 @@ let
         '';
       in {
         Cmd = [ "${entrypoint}/bin/uniworx-entrypoint" ];
+        User = "uniworx:uniworx";
         ExposedPorts = {
           "8080/tcp" = {};
         };

nix/docker/demo-version.json

@@ -1,3 +1,3 @@
 {
-  "version": "0.2.0"
+  "version": "0.3.0"
 }

package.yaml

@@ -253,6 +253,10 @@ when:
     else:
       ghc-options:
         - -O -fllvm
+
+data-files:
+  - testdata/**
+        
 library:
   source-dirs: src
 executables:
@@ -276,6 +280,7 @@ executables:
       - uniworx
     other-modules:
       - Database.Fill
+      - Paths_uniworx
     when:
       - condition: flag(library-only)
         buildable: false

test/Database/Fill.hs

@@ -38,13 +38,15 @@ import qualified Data.Yaml as Yaml
 import Utils.Workflow
 import Utils.Workflow.Lint
 
-import System.Directory (getModificationTime, doesFileExist)
+import System.Directory (getModificationTime, doesFileExist, doesDirectoryExist)
 import System.FilePath.Glob (glob)
 
 import System.IO (hPutStrLn)
 
 import qualified Data.List.NonEmpty as NonEmpty
 
+import Paths_uniworx (getDataFileName)
+
 
 data WorkflowIndexItem = WorkflowIndexItem
   { wiiGraphFile :: FilePath
@@ -59,13 +61,13 @@ deriveJSON defaultOptions
   } ''WorkflowIndexItem
 
 
-testdataDir :: FilePath
+testdataFile :: MonadIO m => FilePath -> m FilePath
-testdataDir = "testdata"
+testdataFile = liftIO . getDataFileName . ("testdata" </>)
-
   
 insertFile :: ( HasFileReference fRef, PersistRecordBackend fRef SqlBackend ) => FileReferenceResidual fRef -> FilePath -> DB (Key fRef)
 insertFile residual fileTitle = do
-  let fileContent = Just . C.sourceFile $ testdataDir </> fileTitle
+  filepath <- testdataFile fileTitle
+  let fileContent = Just $ C.sourceFile filepath
   fileModified <- liftIO getCurrentTime
   sinkFile' File{..} residual >>= insert
 
@@ -1399,8 +1401,6 @@ fillDb = do
         return . pure . AllocationPriorityNumericRecord matr . fromList $ sortOn Down prios
       _other -> return mempty
 
-  liftIO . LBS.writeFile (testdataDir </> "bigAlloc_numeric.csv") $ Csv.encode numericPriorities
-
   ordinalPriorities <- do
     manyUsers' <- shuffleM manyUsers
     flip foldMapM manyUsers' $ \uid -> do
@@ -1410,16 +1410,19 @@ fillDb = do
           return . pure $ Csv.Only matr
         _other -> return mempty
   
-  liftIO . LBS.writeFile (testdataDir </> "bigAlloc_ordinal.csv") $ Csv.encode ordinalPriorities
+  liftIO . handle (\(_ :: IOException) -> return ()) $ do
+    haveTestdata <- doesDirectoryExist "testdata"
+    LBS.writeFile (bool id ("testdata" </>) haveTestdata "bigAlloc_numeric.csv") $ Csv.encode numericPriorities
+    LBS.writeFile (bool id ("testdata" </>) haveTestdata "bigAlloc_ordinal.csv") $ Csv.encode ordinalPriorities
 
-  whenM (liftIO . doesFileExist $ testdataDir </> "workflows" </> "_index.yaml") $ do
+  whenM (liftIO . doesFileExist <=< testdataFile $ "workflows" </> "_index.yaml") $ do
     let displayLinterIssue :: MonadIO m => WorkflowGraphLinterIssue -> m ()
         displayLinterIssue = liftIO . hPutStrLn stderr . displayException
 
-    wfIndex <- Yaml.decodeFileThrow @_ @(Map WorkflowDefinitionName WorkflowIndexItem) $ testdataDir </> "workflows" </> "_index.yaml"
+    wfIndex <- Yaml.decodeFileThrow @_ @(Map WorkflowDefinitionName WorkflowIndexItem) =<< testdataFile ("workflows" </> "_index.yaml")
 
     iforM_ wfIndex $ \wiName WorkflowIndexItem{..} -> handleSql displayLinterIssue $ do
-      graph <- Yaml.decodeFileThrow $ testdataDir </> "workflows" </> wiiGraphFile
+      graph <- Yaml.decodeFileThrow =<< testdataFile ("workflows" </> wiiGraphFile)
       for_ (lintWorkflowGraph graph) $ mapM_ throwM
       workflowDefinitionGraph <- insertSharedWorkflowGraph graph
       let workflowDef = WorkflowDefinition{..}