Merge branch 'zip-to-ddw' into 'cicd'

feat(#8): Deploy as .ddw files intead of zip

See merge request Thoscellen/Wallset!17

.gitignore

@@ -5,3 +5,6 @@
 # Ignore python stuff
 **/__pycache__
 **/.pytest_cache
+
+# Ignore les fichiers système de MacOS
+**/.DS_Store

.gitlab-ci.yml

@@ -1,6 +1,5 @@
 include:
-  - .gitlab/valid_set.yml
+  - .gitlab/*.yml
-  - .gitlab/missing_tag_set.yml
 
 stages:
   - Manifest
@@ -10,7 +9,7 @@ stages:
   - Release
 
 variables:
-  ENTRY_FOLDER: $CI_PROJECT_DIR/test # test directory for testing CI/CD, or assets directory for normal operations
+  ENTRY_FOLDER: test # test directory for testing CI/CD, or assets directory for normal operations
   RELEASE_VERSION: v0.$CI_PIPELINE_IID
   PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE_NAME}/${RELEASE_VERSION}"
 
@@ -19,16 +18,16 @@ variables:
   stage: Manifest
   image: node:latest
   rules:
+    #- if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "cicd"
     - if: $CI_COMMIT_TAG
       when: never
-    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "cicd"
     - changes:
         - ${ENTRY_FOLDER}/${PACKAGE_NAME}/*
   script:
     - cd src/
     - npm install
     - npm run start
-    - cat ${ENTRY_FOLDER}/${PACKAGE_NAME}/theme.json
+    - cat ../${ENTRY_FOLDER}/${PACKAGE_NAME}/theme.json
     - npm run test
   artifacts:
     paths:
@@ -40,7 +39,7 @@ variables:
   stage: Test
   image: python:latest
   rules:
-    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "cicd"
+    #- if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "cicd"
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
         - ${ENTRY_FOLDER}/${PACKAGE_NAME}/*
@@ -54,17 +53,17 @@ variables:
   stage: Bundle
   image: javieraviles/zip:latest
   rules:
+    #- if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "cicd"
     - if: $CI_COMMIT_TAG
       when: never
-    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "cicd"
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
       changes:
         - ${ENTRY_FOLDER}/${PACKAGE_NAME}/*
   script:
-    - zip -j ${PACKAGE_NAME}.zip ${ENTRY_FOLDER}/${PACKAGE_NAME}/*
+    - zip -j ${PACKAGE_NAME}.wdd ${ENTRY_FOLDER}/${PACKAGE_NAME}/*
   artifacts:
     paths:
-      - ${PACKAGE_NAME}.zip
+      - ${PACKAGE_NAME}.ddw
     when: on_success
     expire_in: "1 hour"
 
@@ -72,6 +71,7 @@ variables:
   stage: Distribute
   image: curlimages/curl:latest
   rules:
+    #- if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "cicd"
     - if: $CI_COMMIT_TAG
       when: never
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -81,12 +81,13 @@ variables:
     - |
       curl --fail \
       --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
-      --upload-file ${PACKAGE_NAME}.zip "${PACKAGE_REGISTRY_URL}/${PACKAGE_NAME}.zip?select=package_file"
+      --upload-file ${PACKAGE_NAME}.ddw "${PACKAGE_REGISTRY_URL}/${PACKAGE_NAME}.ddw?select=package_file"
 
 .release-job:
   stage: Release
   image: registry.gitlab.com/gitlab-org/release-cli:latest
   rules:
+    #- if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "cicd"
     - if: $CI_COMMIT_TAG
       when: never
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -95,11 +96,12 @@ variables:
   script:
     - echo "Releasing assets for $PACKAGE_NAME"
   release:
-    name: "Release $PACKAGE_NAME $RELEASE_VERSION"
+    name: "Livrable $PACKAGE_NAME $RELEASE_VERSION"
     tag_name: "$PACKAGE_NAME.$RELEASE_VERSION"
     description: "Version $RELEASE_VERSION"
     ref: "$CI_COMMIT_SHA"
     assets:
       links:
-        - name: ${PACKAGE_NAME}
+        - name: ${PACKAGE_NAME}.ddw
-          url: "${PACKAGE_REGISTRY_URL}/${PACKAGE_NAME}.zip"
+          url: "${PACKAGE_REGISTRY_URL}/${PACKAGE_NAME}.ddw"
+          link_type: package

.gitlab/existing_valid_theme_set.yml

@@ -0,0 +1,32 @@
+manifest-job/existing_valid_theme_set:
+  variables:
+    PACKAGE_NAME: "existing_valid_theme_set"
+  extends: .manifest-job
+
+test-images-job/existing_valid_theme_set:
+  needs:
+    - manifest-job/existing_valid_theme_set
+  variables:
+    PACKAGE_NAME: "existing_valid_theme_set"
+  extends: .test-job
+
+bundle-job/existing_valid_theme_set:
+  needs:
+    - manifest-job/existing_valid_theme_set
+  variables:
+    PACKAGE_NAME: "existing_valid_theme_set"
+  extends: .bundle-job
+
+upload-job/existing_valid_theme_set:
+  needs:
+    - bundle-job/existing_valid_theme_set
+  variables:
+    PACKAGE_NAME: "existing_valid_theme_set"
+  extends: .upload-job
+
+release-job/existing_valid_theme_set:
+  needs:
+    - upload-job/existing_valid_theme_set
+  variables:
+    PACKAGE_NAME: "existing_valid_theme_set"
+  extends: .release-job

.gitlab/unbalanced_set.yml

@@ -0,0 +1,32 @@
+manifest-job/unbalanced_set:
+  variables:
+    PACKAGE_NAME: "unbalanced_set"
+  extends: .manifest-job
+
+test-images-job/unbalanced_set:
+  needs:
+    - manifest-job/unbalanced_set
+  variables:
+    PACKAGE_NAME: "unbalanced_set"
+  extends: .test-job
+
+bundle-job/unbalanced_set:
+  needs:
+    - manifest-job/unbalanced_set
+  variables:
+    PACKAGE_NAME: "unbalanced_set"
+  extends: .bundle-job
+
+upload-job/unbalanced_set:
+  needs:
+    - bundle-job/unbalanced_set
+  variables:
+    PACKAGE_NAME: "unbalanced_set"
+  extends: .upload-job
+
+release-job/unbalanced_set:
+  needs:
+    - upload-job/unbalanced_set
+  variables:
+    PACKAGE_NAME: "unbalanced_set"
+  extends: .release-job

.vscode/launch.json

@@ -12,7 +12,8 @@
 				"start"
 			],
 			"env": {
-				"ENTRY_FOLDER": "${workspaceFolder}/test",
+				"CI_PROJECT_DIR": "${workspaceFolder}",
+				"ENTRY_FOLDER": "test",
 				"PACKAGE_NAME": "valid_set"
 			},
 			"skipFiles": [
@@ -30,7 +31,8 @@
 				"test"
 			],
 			"env": {
-				"ENTRY_FOLDER": "${workspaceFolder}/test",
+				"CI_PROJECT_DIR": "${workspaceFolder}",
+				"ENTRY_FOLDER": "test",
 				"PACKAGE_NAME": "valid_set"
 			},
 			"skipFiles": [
@@ -51,10 +53,69 @@
 				"--color=yes"
 			],
 			"env": {
-				"ENTRY_FOLDER": "${workspaceFolder}/test",
+				"CI_PROJECT_DIR": "${workspaceFolder}",
+				"ENTRY_FOLDER": "test",
 				"PACKAGE_NAME": "valid_set"
 			},
 			"justMyCode": true
 		},
+		{
+			"name": "Gen-Manifest existing_valid_theme_set",
+			"cwd": "${workspaceFolder}/src",
+			"type": "node",
+			"request": "launch",
+			"runtimeExecutable": "npm",
+			"runtimeArgs": [
+				"run",
+				"start"
+			],
+			"env": {
+				"CI_PROJECT_DIR": "${workspaceFolder}",
+				"ENTRY_FOLDER": "test",
+				"PACKAGE_NAME": "existing_valid_theme_set"
+			},
+			"skipFiles": [
+				"<node_internals>/**"
+			],
+		},
+		{
+			"name": "Test-Manifest existing_valid_theme_set",
+			"cwd": "${workspaceFolder}/src",
+			"type": "node",
+			"request": "launch",
+			"runtimeExecutable": "npm",
+			"runtimeArgs": [
+				"run",
+				"test"
+			],
+			"env": {
+				"CI_PROJECT_DIR": "${workspaceFolder}",
+				"ENTRY_FOLDER": "test",
+				"PACKAGE_NAME": "existing_valid_theme_set"
+			},
+			"skipFiles": [
+				"<node_internals>/**"
+			],
+		},
+		{
+			"name": "Test-Images existing_valid_theme_set",
+			"cwd": "${workspaceFolder}/src",
+			"console": "internalConsole",
+			"type": "python",
+			"request": "launch",
+			"module": "pytest",
+			"args": [
+				"test_images.py",
+				"-rA",
+				"--tb=line",
+				"--color=yes"
+			],
+			"env": {
+				"CI_PROJECT_DIR": "${workspaceFolder}",
+				"ENTRY_FOLDER": "test",
+				"PACKAGE_NAME": "existing_valid_theme_set"
+			},
+			"justMyCode": true
+		},
 	]
 }

src/build-manifest.js

@@ -1,18 +1,24 @@
 const fs = require("fs");
 const path = require("path");
 const iptc = require("node-iptc");
+const { exit } = require("process");
 
 
-// find all images
+// Fetch envs to target the right folder
-const slug = process.env["PACKAGE_NAME"];
+const workDir = process.env["CI_PROJECT_DIR"];
 const root = process.env["ENTRY_FOLDER"];
-let workingDirectory = path.join(root, slug);
+const slug = process.env["PACKAGE_NAME"];
+if (!workDir || !slug || !root) exit(1);
+let workingDirectory = path.join(workDir, root, slug);
 
 // extract metadata from them
 fs.readdir(workingDirectory, (err, files) => {
-	if (err) { throw err };
+	if (err) { throw err };	// If IO error, exit with an error message
 
-	// construct a JSON file
+	// If there is already a theme.json silentely exit and let mocha running afterwards
+	if (files.find(file => path.extname(file) == '.json')) exit(0);
+
+	// construct a JSON object
 	let theme = new Object();
 	theme.dayImageList = [];
 	theme.nightImageList = [];
@@ -63,7 +69,7 @@ fs.readdir(workingDirectory, (err, files) => {
 				break;
 		}
 	});
-
+	// make it the theme.json
 	fs.writeFileSync(path.join(workingDirectory, 'theme.json'), JSON.stringify(theme));
 });
 

src/manifest.spec.js

@@ -33,11 +33,13 @@ function flatReferences(manifest) {
 
 describe('Mandatory Checks', function () {
     before(function () {
+        const workDir = process.env["CI_PROJECT_DIR"];
+        expect(workDir).to.be.a("string").that.is.not.empty;
         const slug = process.env["PACKAGE_NAME"];
         expect(slug).to.be.a("string").that.is.not.empty;
         const root = process.env["ENTRY_FOLDER"];
         expect(root).to.be.a("string").that.is.not.empty;
-        workingDirectory = path.join(root, slug);
+        workingDirectory = path.join(workDir, root, slug);
         files = fs.readdirSync(workingDirectory);
         expect(files).to.be.an('array').that.is.not.empty;
         validator = ajv.compile(JSON.parse(fs.readFileSync(themeSchemaFile, 'utf8')));

src/test_images.py

@@ -7,7 +7,7 @@ import os
 import pytest
 
 
-def validate_brightness_image(working_path, theme_config, high_light, image_list, brightness_way):
+def _validate_brightness_image(working_path, theme_config, high_light, image_list, brightness_way):
 	image_pattern = theme_config.get("imageFilename")
 	image_filenames = {}.fromkeys(glob.glob(str(Path(working_path, image_pattern))))
 	# generate an image statistics for each images
@@ -15,6 +15,7 @@ def validate_brightness_image(working_path, theme_config, high_light, image_list
 		this_image = Image.open(an_imagefile).convert("L")
 		this_image_stats = ImageStat.Stat(this_image)
 		image_filenames[an_imagefile] = this_image_stats.mean[0]
+		#print(f'{an_imagefile} : {image_filenames[an_imagefile]}')
 
 	# get the brightest observed image from the list
 	if high_light == "dayHighlight":
@@ -39,9 +40,10 @@ def validate_brightness_image(working_path, theme_config, high_light, image_list
 @pytest.fixture
 def working_path():
 	# get the global variables containing gitlab-given project slug.
+	work_directory = os.environ["CI_PROJECT_DIR"]
+	entry_folder = os.environ["ENTRY_FOLDER"]
 	project_slug = os.environ["PACKAGE_NAME"]
-	root_folder = os.environ["ENTRY_FOLDER"]
+	working_path = Path(work_directory, entry_folder, project_slug)
-	working_path = Path(root_folder, project_slug)
 	if not working_path.is_dir():
 		raise FileNotFoundError(f"No project found for the given {working_path}.")
 	return working_path
@@ -54,11 +56,11 @@ def manifest(working_path):
 
 
 def test_brightest_image(working_path, manifest):
-	validate_brightness_image(working_path, manifest, "dayHighlight", "dayImageList", "Brightest")
+	_validate_brightness_image(working_path, manifest, "dayHighlight", "dayImageList", "Brightest")
 
 
 def test_darkest_image(working_path, manifest):
-	validate_brightness_image(working_path, manifest, "nightHighlight", "nightImageList", "Darkest")
+	_validate_brightness_image(working_path, manifest, "nightHighlight", "nightImageList", "Darkest")
 
 
 def test_image_size(working_path, manifest):

test/existing_valid_theme_set/theme.json

@@ -0,0 +1,13 @@
+{
+	"dayImageList": [
+		1
+	],
+	"imageFilename": "stray__*.jpg",
+	"nightImageList": [
+		2
+	],
+	"imageCredits": "©Thoscellen, ©BlueTwelve",
+	"sunriseImageList": [],
+	"displayName": "Stray",
+	"sunsetImageList": []
+}

test/missing_tag_set/theme.json

@@ -1,7 +1,7 @@
 {
     "imageFilename": "bl3_landscape_*.jpg",
+    "displayName": "BL3",
     "imageCredits": "© Gearbox",
-    "displayName": "",
     "dayImageList": [
         2
     ],