[Piuparts-devel] [Git][debian/piuparts][feature/gitlab-ci] Split salsaci overrides for different stages

[Nicolas Dandrimont (@olasd)]

Nicolas Dandrimont pushed to branch feature/gitlab-ci at Debian / piuparts


Commits:
114923b1 by Nicolas Dandrimont at 2023-05-24T15:44:25+02:00
Split salsaci overrides for different stages

- - - - -


3 changed files:

- .gitlab-ci.yml
- + .gitlab-ci/check-salsaci-overrides
- + .gitlab-ci/salsaci-overrides.yml


Changes:

=====================================
.gitlab-ci.yml
=====================================
@@ -2,6 +2,11 @@
 include:
   - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/salsa-ci.yml
   - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/pipeline-jobs.yml
+  - .gitlab-ci/salsaci-overrides.yml
 
-variables:
-  SALSA_CI_REPROTEST_ENABLE_DIFFOSCOPE: 1
+stages:
+  - lint
+  - salsaci/provisioning
+  - salsaci/build
+  - salsaci/publish
+  - salsaci/test


=====================================
.gitlab-ci/check-salsaci-overrides
=====================================
@@ -0,0 +1,138 @@
+#!/usr/bin/python3
+#
+# Copyright © 2022 Nicolas Dandrimont (nicolas at dandrimont.eu)
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <https://www.gnu.org/licenses/>
+
+from __future__ import annotations
+
+import os
+import sys
+
+import requests
+import yaml
+
+"""Check that salsaci-overrides.yml overrides all of the common salsaci pipelines"""
+
+KNOWN_TOPLEVEL_KEYS = {"include", "variables", "defaults"}
+
+
+def merge(source, destination):
+    """
+    run me with nosetests --with-doctest file.py
+
+    >>> a = { 'first' : { 'all_rows' : { 'pass' : 'dog', 'number' : '1' } } }
+    >>> b = { 'first' : { 'all_rows' : { 'fail' : 'cat', 'number' : '5' } }, 'list': [1, 2] }
+    >>> merge(b, a) == { 'first' : { 'all_rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } }, 'list': [1, 2] }
+    True
+    """
+    for key, value in source.items():
+        if isinstance(value, dict):
+            # get node or create one
+            node = destination.setdefault(key, {})
+            merge(value, node)
+        elif isinstance(value, list):
+            node = destination.setdefault(key, [])
+            destination[key] = destination[key] + value
+        else:
+            destination[key] = value
+
+    return destination
+
+
+def get_gitlabci_config(git_root: str) -> Dict[str, Any]:
+    """Retrieve the full gitlab-ci config out of the git root"""
+
+    gitlabci_path = os.path.join(git_root, ".gitlab-ci.yml")
+    return yaml.safe_load(open(gitlabci_path, "r"))
+
+
+def fetch_config(url: str) -> Dict[str, Any]:
+    """Fetch a gitlab-ci config, resolving any includes"""
+    req = requests.get(url)
+    req.raise_for_status()
+
+    config = yaml.safe_load(req.content)
+
+    merged_config = {}
+    for include in config.pop("include", []):
+        included_config = fetch_config(include)
+        merge(included_config, merged_config)
+
+    merge(config, merged_config)
+    return merged_config
+
+
+def get_salsaci_config(gitlabci_config: Dict[str, Any]) -> Dict[str, Any]:
+    """Retrieve the config of the salsaci jobs, out of .gitlab-ci.yml includes"""
+    includes = gitlabci_config.get("include", [])
+    if not includes:
+        raise ValueError("No includes found in .gitlab-ci.yml")
+
+    for include in includes:
+        if include.endswith("pipeline-jobs.yml"):
+            break
+    else:
+        raise ValueError("No include for pipeline-jobs.yml found in .gitlab-ci.yml")
+
+    return fetch_config(include)
+
+
+def get_salsaci_overrides(
+    git_root: str, gitlabci_config: Dict[str, Any]
+) -> Dict[str, Any]:
+    """Retrieve the salsaci overrides, out of .gitlab-ci.yml includes"""
+    includes = gitlabci_config.get("include", [])
+    if not includes:
+        raise ValueError("No includes found in .gitlab-ci.yml")
+
+    for include in includes:
+        if include.endswith("salsaci-overrides.yml"):
+            break
+    else:
+        raise ValueError("No include for salsaci-overrides.yml found in .gitlab-ci.yml")
+
+    return yaml.safe_load(open(os.path.join(git_root, include), "r"))
+
+
+if __name__ == "__main__":
+    git_root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), ".."))
+    gitlabci_config = get_gitlabci_config(git_root)
+    salsaci_config = get_salsaci_config(gitlabci_config)
+    salsaci_overrides = get_salsaci_overrides(git_root, gitlabci_config)
+
+    keys_missing = {
+        key
+        for key in salsaci_config.keys()
+        - salsaci_overrides.keys()
+        - KNOWN_TOPLEVEL_KEYS
+        if not key.startswith(".")
+    }
+
+    for key in keys_missing:
+        print("missing", key, salsaci_config[key])
+
+    keys_extra = {
+        key
+        for key in salsaci_overrides.keys()
+        - salsaci_config.keys()
+        - KNOWN_TOPLEVEL_KEYS
+        if salsaci_overrides[key]["stage"].startswith("salsaci/")
+    }
+
+    for key in keys_extra:
+        print("extra", key, salsaci_overrides[key])
+
+    if keys_missing or keys_extra:
+        sys.exit(1)


=====================================
.gitlab-ci/salsaci-overrides.yml
=====================================
@@ -0,0 +1,52 @@
+---
+check-salsaci-overrides:
+  stage: lint
+  image: debian:bullseye
+  before_script:
+    - apt-get update
+    - apt-get -y dist-upgrade
+    - apt-get -y install python3 python3-requests python3-yaml
+  script:
+    - ./.gitlab-ci/check-salsaci-overrides
+
+extract-source:
+  stage: salsaci/provisioning
+build:
+  stage: salsaci/build
+build i386:
+  stage: salsaci/build
+build arm64:
+  stage: salsaci/build
+build armel:
+  stage: salsaci/build
+build armhf:
+  stage: salsaci/build
+build source:
+  stage: salsaci/build
+aptly:
+  stage: salsaci/publish
+test-build-any:
+  stage: salsaci/test
+test-build-all:
+  stage: salsaci/test
+reprotest:
+  stage: salsaci/test
+atomic-reprotest:
+  stage: salsaci/test
+lintian:
+  stage: salsaci/test
+piuparts:
+  stage: salsaci/test
+autopkgtest:
+  stage: salsaci/test
+blhc:
+  stage: salsaci/test
+rc-bugs:
+  stage: salsaci/test
+missing-breaks:
+  stage: salsaci/test
+test-crossbuild-arm64:
+  stage: salsaci/test
+
+variables:
+  SALSA_CI_REPROTEST_ENABLE_DIFFOSCOPE: 1



View it on GitLab: https://salsa.debian.org/debian/piuparts/-/commit/114923b1142611f66cdab3fb2b1d66ef38d3031b

-- 
View it on GitLab: https://salsa.debian.org/debian/piuparts/-/commit/114923b1142611f66cdab3fb2b1d66ef38d3031b
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/piuparts-devel/attachments/20230524/73581ee4/attachment-0001.htm>