diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..b718f72 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,29 @@ +--- +name: Checks + +on: + push: + branches: [main] + pull_request: {} + +permissions: {} + +jobs: + python: + name: Lint Python & validate catalog + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Install tools + run: pip install --quiet pyyaml ruff + - name: Ruff (scripts) + run: ruff check scripts/ + - name: Ruff (apps) + run: ruff check apps/ + - name: Validate catalog consistency + run: python3 scripts/check-catalog.py diff --git a/README.md b/README.md index 31c6636..c0143fd 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,39 @@ Configuration of our testing Jenkins (https://testing.stackable.tech) This catalog contains the config data for the tests run in https://testing.stackable.tech. -* [platforms.yaml](catalog/platforms.yaml) contains the definition of the test system vendors and platforms +* [platforms.yaml](catalog/platforms.yaml) contains the definition of the test system vendors and platforms (do not hand edit this, see below) * [operator-tests.yaml](catalog/operator-tests.yaml) defines the integration tests for our operators See this [Nuclino page](https://app.nuclino.com/Stackable/Engineering/Configuring-Test-Jobs-4e993d84-19b4-4081-846d-738f9f38573d) for further information. +### Updating the catalog + +**Do not hand edit `platforms.yaml`** + +[`scripts/update-platforms.py`](scripts/update-platforms.py) refreshes the Kubernetes versions in `platforms.yaml`. +It fetches the latest versions from the Replicated and IONOS APIs, keeps the latest patch per minor version line, adds ARM variants where supported, and rewrites the file. + +Prerequisites: + +* Python 3 with `PyYAML` (`pip install pyyaml`) +* [`replicated`](https://docs.replicated.com/reference/replicated-cli-installing) CLI, authenticated (`REPLICATED_API_TOKEN`) +* [`ionosctl`](https://github.com/ionos-cloud/ionosctl) CLI, authenticated + +Run it from the repository root and review the diff before committing: + +```bash +python3 scripts/update-platforms.py +git diff catalog/platforms.yaml +``` + +[`scripts/check-catalog.py`](scripts/check-catalog.py) validates that `platforms.yaml` and `operator-tests.yaml` are consistent (every platform referenced by a test exists, every platform has a known provider). +It exits non-zero on errors, so it is suitable for CI. +Run it after editing either catalog file: + +```bash +python3 scripts/check-catalog.py +``` + ## Apps Under [apps/](apps/README.md), we maintain a bunch of Dockerized applications for the maintenance of Jenkins and to run the tests. diff --git a/apps/src/modules/catalog.py b/apps/src/modules/catalog.py index 5f53bc5..aca9c44 100644 --- a/apps/src/modules/catalog.py +++ b/apps/src/modules/catalog.py @@ -36,7 +36,6 @@ def read_platforms(logger): logger logger (String-consuming function) """ global platforms - platforms = hiyapyco.load("/platforms.yaml")["platforms"] platforms_yaml = hiyapyco.load("/platforms.yaml") if "platforms" not in platforms_yaml: logger("platforms.yaml does not contain platforms.") diff --git a/catalog/platforms.yaml b/catalog/platforms.yaml index 81c551e..6ae5f62 100644 --- a/catalog/platforms.yaml +++ b/catalog/platforms.yaml @@ -9,11 +9,11 @@ platforms: instance-type: r1.xlarge node-count: 1 versions: - - 1.35.0 - - 1.34.3 - - 1.33.7 + - 1.36.1 + - 1.35.5 + - 1.34.8 + - 1.33.12 - 1.32.11 - - 1.31.14 - id: replicated-kind-arm name: kind on replicated.com - ARM provider: replicated @@ -23,11 +23,11 @@ platforms: instance-type: r1a.xlarge node-count: 1 versions: - - 1.35.0 - - 1.34.3 - - 1.33.7 + - 1.36.1 + - 1.35.5 + - 1.34.8 + - 1.33.12 - 1.32.11 - - 1.31.14 - id: replicated-rke2 name: RKE2 on replicated.com provider: replicated @@ -37,10 +37,10 @@ platforms: instance-type: r1.large node-count: 3 versions: - - 1.35.0 - - 1.34.3 - - 1.33.7 - - 1.32.11 + - 1.35.6 + - 1.34.9 + - 1.33.13 + - 1.32.13 - id: replicated-rke2-arm name: RKE2 on replicated.com - ARM provider: replicated @@ -50,10 +50,10 @@ platforms: instance-type: r1a.large node-count: 3 versions: - - 1.35.0 - - 1.34.3 - - 1.33.7 - - 1.32.11 + - 1.35.6 + - 1.34.9 + - 1.33.13 + - 1.32.13 - id: replicated-k3s name: k3s on replicated.com provider: replicated @@ -63,10 +63,11 @@ platforms: instance-type: r1.large node-count: 3 versions: - - 1.35.0 - - 1.34.3 - - 1.33.7 - - 1.32.11 + - 1.36.2 + - 1.35.6 + - 1.34.9 + - 1.33.13 + - 1.32.13 - id: replicated-k3s-arm name: k3s on replicated.com - ARM provider: replicated @@ -76,10 +77,11 @@ platforms: instance-type: r1a.large node-count: 3 versions: - - 1.35.0 - - 1.34.3 - - 1.33.7 - - 1.32.11 + - 1.36.2 + - 1.35.6 + - 1.34.9 + - 1.33.13 + - 1.32.13 - id: replicated-azure name: Azure AKS on replicated.com provider: replicated @@ -89,9 +91,10 @@ platforms: instance-type: Standard_D4S_v5 node-count: 3 versions: + - "1.36" + - "1.35" - "1.34" - "1.33" - - "1.32" - id: replicated-azure-arm name: Azure AKS on replicated.com - ARM provider: replicated @@ -101,9 +104,10 @@ platforms: instance-type: Standard_D4ps_v5 node-count: 3 versions: + - "1.36" + - "1.35" - "1.34" - "1.33" - - "1.32" - id: replicated-aws name: Amazon AWS EKS on replicated.com provider: replicated @@ -113,11 +117,11 @@ platforms: instance-type: m6i.large node-count: 3 versions: + - "1.36" - "1.35" - "1.34" - "1.33" - "1.32" - - "1.31" - id: replicated-aws-arm name: Amazon AWS EKS on replicated.com - ARM provider: replicated @@ -127,11 +131,11 @@ platforms: instance-type: m7g.large node-count: 3 versions: + - "1.36" - "1.35" - "1.34" - "1.33" - "1.32" - - "1.31" - id: replicated-gke name: Google GKE on replicated.com provider: replicated @@ -141,6 +145,8 @@ platforms: instance-type: e2-standard-2 node-count: 3 versions: + - "1.36" + - "1.35" - "1.34" - "1.33" - "1.32" @@ -153,6 +159,8 @@ platforms: instance-type: t2a-standard-2 node-count: 3 versions: + - "1.36" + - "1.35" - "1.34" - "1.33" - "1.32" @@ -164,7 +172,7 @@ platforms: distribution: openshift instance-type: r1.large node-count: 3 - versions: &id001 + versions: - 4.22.0-okd - 4.21.0-okd - 4.20.0-okd @@ -178,7 +186,12 @@ platforms: distribution: openshift instance-type: r1a.large node-count: 3 - versions: *id001 + versions: + - 4.22.0-okd + - 4.21.0-okd + - 4.20.0-okd + - 4.19.0-okd + - 4.18.0-okd - id: ionos-k8s name: IONOS managed K8s provider: ionos @@ -190,9 +203,9 @@ platforms: node-count: 3 ram: 8192 versions: + - 1.34.2 - 1.33.3 - - 1.32.6 - - 1.31.10 + - 1.32.7 - id: replicated-oke name: OKE on replicated.com provider: replicated @@ -202,9 +215,9 @@ platforms: instance-type: VM.Standard3.Flex.2 node-count: 3 versions: + - 1.35.2 - 1.34.2 - - 1.33.1 - - 1.32.10 + - 1.33.10 - id: replicated-oke-arm name: OKE on replicated.com - ARM provider: replicated @@ -214,9 +227,9 @@ platforms: instance-type: VM.Standard.A1.Flex.2 node-count: 3 versions: + - 1.35.2 - 1.34.2 - - 1.33.1 - - 1.32.10 + - 1.33.10 providers: - id: replicated name: replicated.com diff --git a/scripts/check-catalog.py b/scripts/check-catalog.py new file mode 100755 index 0000000..b776d40 --- /dev/null +++ b/scripts/check-catalog.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +""" +Validate the catalog files for internal consistency. + +platforms.yaml and operator-tests.yaml are maintained separately (the former by +update-platforms.py, the latter by hand), so they can drift apart. This check is +meant to run in CI so that drift is caught at edit time instead of surfacing +mid-test as a confusing "platform does not exist" error. + +Checks: +1. Every platform's `provider` refers to a provider defined in platforms.yaml. +2. Every platform ID referenced by an operator test exists in platforms.yaml + (a dangling reference is an error). +3. Every platform defined in platforms.yaml is used by at least one operator + test (an unused platform is a warning, not an error). + +Exits non-zero if any error-level check fails. +""" + +import sys +from pathlib import Path +from typing import List + +import yaml + + +def load_yaml(path: str): + try: + with open(path) as f: + return yaml.safe_load(f) + except FileNotFoundError: + print(f"❌ File not found: {path}") + sys.exit(1) + except yaml.YAMLError as e: + print(f"❌ Failed to parse {path}: {e}") + sys.exit(1) + + +def check_catalog( + platforms_path: str = "catalog/platforms.yaml", + tests_path: str = "catalog/operator-tests.yaml", +) -> bool: + """Return True if the catalog is consistent, False if there are errors.""" + print("=" * 60) + print("Catalog Consistency Check") + print("=" * 60) + + platforms_yaml = load_yaml(platforms_path) + operator_tests = load_yaml(tests_path) or [] + + platforms = platforms_yaml.get("platforms") or [] + providers = platforms_yaml.get("providers") or [] + provider_ids = {p["id"] for p in providers} + platform_ids = {p["id"] for p in platforms} + + errors: List[str] = [] + warnings: List[str] = [] + + # 1. Each platform's provider must be defined. + for platform in platforms: + provider = platform.get("provider") + if provider not in provider_ids: + errors.append( + f"platform '{platform.get('id')}' has unknown provider " + f"'{provider}' (known providers: {sorted(provider_ids)})" + ) + + # 2. Operator tests may only reference platforms that exist. + referenced_ids = set() + for operator_test in operator_tests: + for platform_ref in operator_test.get("platforms", []): + ref_id = platform_ref.get("id") + referenced_ids.add(ref_id) + if ref_id not in platform_ids: + errors.append( + f"operator test '{operator_test.get('id')}' references " + f"platform '{ref_id}' which is not defined in {platforms_path}" + ) + + # 3. Platforms that no test uses are dead weight (warn only). + for unused in sorted(platform_ids - referenced_ids): + warnings.append( + f"platform '{unused}' is defined but not used by any operator test" + ) + + for warning in warnings: + print(f"⚠️ {warning}") + for error in errors: + print(f"❌ {error}") + + print("-" * 60) + if errors: + print(f"❌ Catalog validation FAILED with {len(errors)} error(s).") + return False + + print( + f"✓ Catalog OK: {len(platform_ids)} platforms, " + f"{len(operator_tests)} operator tests, {len(warnings)} warning(s)." + ) + return True + + +def main(): + # Allow running from anywhere as long as the catalog paths resolve. + if not Path("catalog/platforms.yaml").exists(): + print("❌ Run this from the repository root (catalog/ not found).") + sys.exit(1) + + if not check_catalog(): + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/update-platforms.py b/scripts/update-platforms.py index 022433b..8fc248e 100755 --- a/scripts/update-platforms.py +++ b/scripts/update-platforms.py @@ -7,11 +7,15 @@ 2. Fetches available versions from `ionosctl k8s version list --output json` 3. Filters to keep only the latest patch version from each minor version line 4. Adds ARM alternatives where supported (Replicated only) -5. Updates the platforms.yaml file while preserving structure -6. Formats the YAML output using yq for consistency +5. Rewrites platforms.yaml in a canonical, deterministic style + +The script owns the layout of platforms.yaml end to end (see _CatalogDumper), +so it needs no external formatter and re-running it on an unchanged catalog +produces no diff. Do not hand-edit platforms.yaml; re-run this script instead. """ import json +import shutil import subprocess import sys from collections import defaultdict @@ -21,6 +25,42 @@ import yaml +class _CatalogDumper(yaml.SafeDumper): + """Emit YAML in the catalog's canonical style so the script fully owns the file. + + Two deviations from PyYAML's defaults keep diffs limited to real changes: + - Block sequences are indented under their key (the committed 2-space style) + rather than sitting at the parent's indentation. + - Anchors/aliases are never emitted: the script owns the whole file, so every + list is written literally. This keeps entries independent (e.g. OpenShift + and its ARM variant) instead of coupling them through a shared anchor. + """ + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, indentless=False) + + def ignore_aliases(self, data): + return True + + +def _represent_str(dumper: yaml.Dumper, data: str): + """Double-quote scalars that would otherwise parse as a non-string. + + Version lines like "1.34" must stay strings (bare 1.34 is a float). We defer + the "would this parse as a number/bool/etc.?" decision to PyYAML's own + resolver, and only override the quote character to double quotes to match the + committed convention. Genuine strings (ids, names, "1.35.0") stay unquoted. + """ + node = dumper.represent_scalar("tag:yaml.org,2002:str", data) + resolved = dumper.resolve(yaml.nodes.ScalarNode, data, (True, False)) + if resolved != "tag:yaml.org,2002:str": + node.style = '"' + return node + + +_CatalogDumper.add_representer(str, _represent_str) + + class PlatformUpdater: """Updates platforms.yaml with latest versions from Replicated and IONOS.""" @@ -68,10 +108,26 @@ class PlatformUpdater: # Distributions to exclude EXCLUDED_DISTRIBUTIONS = {"embedded-cluster"} + # Keep at most this many (latest) minor version lines per platform. + MAX_VERSIONS_PER_PLATFORM = 5 + def __init__(self, platforms_yaml_path: str = "catalog/platforms.yaml"): self.platforms_yaml_path = Path(platforms_yaml_path) self.replicated_data = None + def check_prerequisites(self) -> None: + """Verify the provider CLIs used to fetch versions are available.""" + print("🔎 Checking prerequisites...") + + # The version data comes from these provider CLIs; fail early with a + # clear message rather than midway through the update. + for tool in ("replicated", "ionosctl"): + if shutil.which(tool) is None: + print(f"❌ `{tool}` not found on PATH. It is required to fetch versions.") + sys.exit(1) + + print("✓ Prerequisites OK") + def fetch_replicated_versions(self) -> List[Dict]: """Fetch available versions from Replicated API.""" print("📡 Fetching versions from Replicated API...") @@ -104,6 +160,24 @@ def fetch_ionos_versions(self) -> List[str]: check=True, ) data = json.loads(result.stdout) + + # `ionosctl ... --output json` does not return a JSON array. It + # returns a JSON-encoded *string* holding a Go slice literal, e.g. + # "[1.34.2 1.33.3 1.32.7]" + # Iterating that string directly (as this script used to) walks it + # character by character and produces garbage versions like "7", + # "6", ... — which silently wiped/corrupted the IONOS entry. Parse + # the slice literal into an actual list of version strings. + if isinstance(data, str): + data = data.strip().strip("[]").split() + + if not isinstance(data, list) or not all( + self.parse_version(v) != (0, 0, 0) for v in data + ): + print(f"❌ Unexpected IONOS version data: {data!r}") + print(" Expected a list of versions like ['1.34.2', '1.33.3'].") + sys.exit(1) + print(f"✓ Fetched {len(data)} versions from IONOS") return data except subprocess.CalledProcessError as e: @@ -187,10 +261,7 @@ def update_platform_versions(self, platform: Dict, distribution_data: Dict) -> D distribution_data["versions"], distribution ) - # Limit to reasonable number of versions (e.g., last 5 minor versions) - max_versions = 5 - if len(latest_versions) > max_versions: - latest_versions = latest_versions[:max_versions] + latest_versions = latest_versions[: self.MAX_VERSIONS_PER_PLATFORM] platform["versions"] = latest_versions return platform @@ -205,13 +276,17 @@ def create_arm_variant(self, platform: Dict, suffix: str = "-arm") -> Dict | Non if not arm_instance: return None - # Create a copy with ARM-specific changes + # Create a copy with ARM-specific changes. Copy the nested spec and + # versions too, so the ARM variant never shares mutable state with its + # x86 counterpart. arm_platform = platform.copy() arm_platform["id"] = platform["id"] + suffix # Use "-" instead of parentheses to avoid parsing issues in Jenkins arm_platform["name"] = platform["name"] + " - ARM" arm_platform["spec"] = platform["spec"].copy() arm_platform["spec"]["instance-type"] = arm_instance + if "versions" in platform: + arm_platform["versions"] = list(platform["versions"]) return arm_platform @@ -256,9 +331,7 @@ def update_platforms(self) -> Dict: latest_versions = self.get_latest_patch_versions( ionos_versions, "ionos" ) - # Limit to 5 versions - if len(latest_versions) > 5: - latest_versions = latest_versions[:5] + latest_versions = latest_versions[: self.MAX_VERSIONS_PER_PLATFORM] platform["versions"] = latest_versions print(f" ✓ Updated {platform_id}") @@ -315,7 +388,13 @@ def update_platforms(self) -> Dict: arm_id ) # Track it to avoid future duplicates - # Check for missing distributions from Replicated and add them + # Warn about distributions Replicated offers that aren't in the catalog. + # We deliberately do NOT auto-create them: the correct instance-type, + # node-count and disk-size are provider-specific (a cloud distribution + # like eks/aks/gke cannot use a Replicated `r1.*` instance type), so a + # fabricated entry would be wrong and fail at test time. Adding a new + # distribution is a rare, deliberate act — surface it and let a human + # add it with the right spec. print("\n🔍 Checking for missing Replicated distributions...") for short_name, dist_data in distribution_lookup.items(): if self.should_skip_distribution(short_name): @@ -325,43 +404,11 @@ def update_platforms(self) -> Dict: if not platform_id or platform_id in existing_platform_ids: continue # Already exists or not mapped - # Create new platform for this distribution - # Generate name if not provided (rke2, oke don't have "name" field) - platform_name = dist_data.get("name", f"{short_name.upper()} on replicated.com") - print(f" ➕ Adding new platform: {platform_id} ({platform_name})") - - # Get versions - latest_versions = self.get_latest_patch_versions( - dist_data["versions"], short_name + print( + f" ⚠️ Replicated offers distribution '{short_name}' " + f"(would be '{platform_id}') which is not in {self.platforms_yaml_path}.\n" + f" Add it manually with the correct spec if it should be tested." ) - if len(latest_versions) > 5: - latest_versions = latest_versions[:5] - - new_platform = { - "id": platform_id, - "name": platform_name, - "provider": "replicated", - "spec": { - "distribution": short_name, - "instance-type": "r1.xlarge", # Default instance type - "node-count": 3, - "disk-size": 100, - }, - "versions": latest_versions, - } - - updated_platforms.append(new_platform) - existing_platform_ids.add(platform_id) - - # Create ARM variant - arm_platform = self.create_arm_variant(new_platform) - if arm_platform: - arm_id = arm_platform["id"] - print( - f" ➕ Adding ARM variant: {arm_id} (instance: {arm_platform['spec']['instance-type']})" - ) - new_arm_platforms.append(arm_platform) - existing_platform_ids.add(arm_id) # Add new ARM platforms after their x86 counterparts final_platforms = [] @@ -381,32 +428,27 @@ def update_platforms(self) -> Dict: return platforms_data def save_platforms(self, data: Dict) -> None: - """Save updated platforms to YAML file.""" + """Save updated platforms to YAML file. + + The output is produced entirely by `_CatalogDumper`, which emits the + catalog's canonical style directly. There is deliberately no external + formatter (previously `yq`): the script owns the format on its own, so + the result is deterministic regardless of what tools are installed. + """ print(f"\n💾 Saving updated platforms to {self.platforms_yaml_path}...") try: with open(self.platforms_yaml_path, "w") as f: - yaml.safe_dump( + yaml.dump( data, f, + Dumper=_CatalogDumper, sort_keys=True, allow_unicode=True, explicit_start=True, + default_flow_style=False, indent=2, ) print("✓ Saved successfully") - - # Fix YAML formatting using yq (PyYAML has known formatting issues) - # https://github.com/yaml/pyyaml/issues/234 - print("🔧 Fixing YAML formatting with yq...") - result = subprocess.run( - ["yq", "-i", "-P", str(self.platforms_yaml_path)], - capture_output=True, - text=True, - ) - if result.returncode == 0: - print("✓ YAML formatting fixed") - else: - print(f"⚠️ yq formatting failed (non-critical): {result.stderr}") except Exception as e: print(f"❌ Failed to save: {e}") sys.exit(1) @@ -417,6 +459,7 @@ def run(self) -> None: print("Platform Version Updater") print("=" * 60) + self.check_prerequisites() updated_data = self.update_platforms() self.save_platforms(updated_data)