Unverified Commit 41352e60 authored by Artemis Tosini's avatar Artemis Tosini Committed by GitHub
Browse files

freebsd.source: Use hashes from `git archive` (#397513)

parents 219272b3 e6b11fe5

pkgs/os-specific/bsd/freebsd/pkgs/source.nix

+4 −8
Original line number Diff line number Diff line 
{ fetchFromGitHub, sourceData }:



# Using fetchFromGitHub from their mirror because it's a lot faster than their git server

# If you want you could fetchgit from "https://git.FreeBSD.org/src.git" instead.

# Using fetchFromGitHub from their mirror because we cannot use git during bootstrap

# If you want you could fetchurl from "https://cgit.FreeBSD.org/src" instead.

# The update script still pulls directly from git.freebsd.org

# Note that hashes with `forceFetchGit = true` may be different due to `export-ignore`

# directives in gitattributes files.

fetchFromGitHub {

  owner = "freebsd";

  repo = "freebsd-src";

  inherit (sourceData) rev hash;



  # The GitHub export excludes some files in the git source

  # that were marked `export-ignore`.

  # A normal git checkout will keep those files,

  # matching the update script

  forceFetchGit = true;

}

pkgs/os-specific/bsd/freebsd/update.py

+28 −33
Original line number Diff line number Diff line
@@ -11,12 +11,13 @@ import pandas 
import re

import subprocess

import sys

import tarfile

import tempfile

import typing

import urllib.request



_QUERY_VERSION_PATTERN = re.compile('^([A-Z]+)="(.+)"$')

_RELEASE_PATCH_PATTERN = re.compile('^RELEASE-p([0-9]+)$')

_RELEASE_PATCH_PATTERN = re.compile("^RELEASE-p([0-9]+)$")

BASE_DIR = os.path.dirname(os.path.abspath(__file__))

MIN_VERSION = packaging.version.Version("13.0.0")

MAIN_BRANCH = "main"
@@ -32,7 +33,7 @@ BRANCH_PATTERN = re.compile( 


def request_supported_refs() -> list[str]:

    # Looks pretty shady but I think this should work with every version of the page in the last 20 years

    r = re.compile("^h\d$", re.IGNORECASE)

    r = re.compile(r"^h\d$", re.IGNORECASE)

    soup = bs4.BeautifulSoup(

        urllib.request.urlopen("https://www.freebsd.org/security"), features="lxml"

    )
@@ -45,11 +46,11 @@ def request_supported_refs() -> list[str]: 
    return list(df["Branch"])





def query_version(repo: git.Repo) -> dict[str, typing.Any]:

def query_version(work_dir: str) -> dict[str, typing.Any]:

    # This only works on FreeBSD 13 and later

    text = (

        subprocess.check_output(

            ["bash", os.path.join(repo.working_dir, "sys", "conf", "newvers.sh"), "-v"]

            ["bash", os.path.join(work_dir, "sys", "conf", "newvers.sh"), "-v"]

        )

        .decode("utf-8")

        .strip()
@@ -86,17 +87,22 @@ def handle_commit( 
        print(f"{ref_name}: revision still {rev.hexsha}, skipping")

        return old_versions[ref_name]



    repo.git.checkout(rev)

    print(f"{ref_name}: checked out {rev.hexsha}")

    tar_content = io.BytesIO()

    repo.archive(tar_content, rev, format="tar")

    tar_content.seek(0)



    with tempfile.TemporaryDirectory(dir="/tmp") as work_dir:

        file = tarfile.TarFile(fileobj=tar_content)

        file.extractall(path=work_dir, filter="data")



        full_hash = (

        subprocess.check_output(["nix", "hash", "path", "--sri", repo.working_dir])

            subprocess.check_output(["nix", "hash", "path", "--sri", work_dir])

            .decode("utf-8")

            .strip()

        )

        print(f"{ref_name}: hash is {full_hash}")



    version = query_version(repo)

        version = query_version(work_dir)

        print(f"{ref_name}: version is {version['version']}")



    return {
@@ -115,30 +121,19 @@ def main() -> None: 
    print(f"Selected temporary directory {temp_dir.name}")



    if len(sys.argv) >= 2:

        orig_repo = git.Repo(sys.argv[1])

        print(f"Fetching updates on {orig_repo.git_dir}")

        orig_repo.remote("origin").fetch()

        repo = git.Repo(sys.argv[1])

        print(f"Fetching updates on {repo.git_dir}")

        repo.remote("origin").fetch()

    else:

        print("Cloning source repo")

        orig_repo = git.Repo.clone_from(

            "https://git.FreeBSD.org/src.git", to_path=os.path.join(temp_dir.name, "orig")

        repo = git.Repo.clone_from(

            "https://git.FreeBSD.org/src.git", to_path=temp_dir.name

        )



    supported_refs = request_supported_refs()

    print(f"Supported refs are: {' '.join(supported_refs)}")



    print("Doing git crimes, do not run `git worktree prune` until after script finishes!")

    workdir = os.path.join(temp_dir.name, "work")

    git.cmd.Git(orig_repo.git_dir).worktree("add", "--orphan", workdir)



    # Have to create object before removing .git otherwise it will complain

    repo = git.Repo(workdir)

    repo.git.set_persistent_git_options(git_dir=repo.git_dir)

    # Remove so that nix hash doesn't see the file

    os.remove(os.path.join(workdir, ".git"))



    print(f"Working in directory {repo.working_dir} with git directory {repo.git_dir}")



    print(f"git directory {repo.git_dir}")



    try:

        with open(os.path.join(BASE_DIR, "versions.json"), "r") as f:
@@ -191,10 +186,10 @@ def main() -> None: 
        )

        versions[fullname] = result





    with open(os.path.join(BASE_DIR, "versions.json"), "w") as out:

        json.dump(versions, out, sort_keys=True, indent=2)

        out.write("\n")



if __name__ == '__main__':



if __name__ == "__main__":

    main()

pkgs/os-specific/bsd/freebsd/versions.json

+10 −10
Original line number Diff line number Diff line 
{

  "main": {

    "hash": "sha256-xsgY5Ex/B5ngOTa5OZRauSaSYvET5lWI7veJRrSq1oY=",

    "hash": "sha256-f69U2FX+3GQimdUJunWApFseeKJhzjgNw4h26ZBPiT0=",

    "ref": "main",

    "refType": "branch",

    "rev": "c5773d366ecc5271b9bd6e5506c00fb3520f19ae",

    "rev": "9357c694e8dca627c25b15529e8435b2ab3dd48b",

    "supported": false,

    "version": {

      "branch": "CURRENT",

      "major": 15,

      "minor": 0,

      "reldate": "1500035",

      "reldate": "1500036",

      "release": "15.0-CURRENT",

      "revision": "15.0",

      "type": "FreeBSD",
@@ -161,7 +161,7 @@ 
    }

  },

  "release/14.2.0": {

    "hash": "sha256-qZkeuUZbuPOvXZBgP5x6Hii1YN7XdDJzwZeYacIR5BI=",

    "hash": "sha256-q5/0pKRlmzZUN+VMwUImkve+NTT1+O68YlwgXp2/aQM=",

    "ref": "release/14.2.0",

    "refType": "tag",

    "rev": "c8918d6c7412fce87922e9bd7e4f5c7d7ca96eb7",
@@ -308,7 +308,7 @@ 
    "ref": "releng/14.1",

    "refType": "branch",

    "rev": "f389e68ca980b7e053a34d9eddde89b4c2a1ee6c",

    "supported": true,

    "supported": false,

    "version": {

      "branch": "RELEASE-p8",

      "major": 14,
@@ -322,7 +322,7 @@ 
    }

  },

  "releng/14.2": {

    "hash": "sha256-XP8BFnXvziaC9wOJj8q31UZXFqCUE7WQ5FdJHEZWGbg=",

    "hash": "sha256-iSeTif/cEkqhIQ5hSNF+Rx7dORU13Bc0Dk8Zv4TYWtA=",

    "ref": "releng/14.2",

    "refType": "branch",

    "rev": "ac2cbb46b5f1efa7f7b5d4eb15631337329ec5b2",
@@ -340,10 +340,10 @@ 
    }

  },

  "stable/13": {

    "hash": "sha256-J9SJKeR6Den3Sep2o4r0cqIDd2V5gbY0Ow9eP69Ny0o=",

    "hash": "sha256-rex3CUXyyNDz/TouNqlzZFo6DAwSOKXTdC5HaRQoKKc=",

    "ref": "stable/13",

    "refType": "branch",

    "rev": "a8431b47adae8f8b731206dc38d82b2245ad245e",

    "rev": "64f5a71c1cb79c09e50a37ddbe958224bb64add4",

    "supported": true,

    "version": {

      "branch": "STABLE",
@@ -357,10 +357,10 @@ 
    }

  },

  "stable/14": {

    "hash": "sha256-tleB6J5Cg1SIN2LCfvV3Cfp4Lxx65UHmiILpin6UYGY=",

    "hash": "sha256-s+pj8LttAnKNNohUsJNOJRFmLg6x06tlkJn3xQnFQiY=",

    "ref": "stable/14",

    "refType": "branch",

    "rev": "6e510d8fbaf8d91da235fe28250cd48124edda9f",

    "rev": "e6a470ffcbd708cf404472bb871c2cb76eaa7b39",

    "supported": true,

    "version": {

      "branch": "STABLE",