From 1a017cfac704bc2efa43bf92e9010b7db89ae518 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 1 Sep 2023 16:12:33 -0400 Subject: [PATCH 01/17] Add more task templates to the blank role. --- ansible/roles/blank/tasks/common.yaml | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/ansible/roles/blank/tasks/common.yaml b/ansible/roles/blank/tasks/common.yaml index fef1101..386bbf1 100644 --- a/ansible/roles/blank/tasks/common.yaml +++ b/ansible/roles/blank/tasks/common.yaml @@ -1,3 +1,35 @@ +# - name: Create directories +# file: +# name: "{{ item }}" +# state: directory +# mode: 0755 +# owner: root +# group: wheel +# loop: +# - /foo/bar + +# - name: Install scripts +# copy: +# src: "files/{{ item.src }}" +# dest: "{{ item.dest }}" +# mode: 0755 +# owner: root +# group: wheel +# loop: +# - src: foo.bash +# dest: /usr/local/bin/foo + +# - name: Install Configuration +# copy: +# src: "files/{{ item.src }}" +# dest: "{{ item.dest }}" +# mode: 0600 +# owner: root +# group: wheel +# loop: +# - src: foo.conf +# dest: /usr/local/etc/foo.conf + - import_tasks: tasks/freebsd.yaml when: 'os_flavor == "freebsd"' From 112cba2b8c5bf157ead60e80188464e3db8b6db5 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 1 Sep 2023 16:12:16 -0400 Subject: [PATCH 02/17] Add more ports to be built along with the kernel. --- ansible/roles/build/templates/src.conf.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/build/templates/src.conf.j2 b/ansible/roles/build/templates/src.conf.j2 index 9869264..75f0dec 100644 --- a/ansible/roles/build/templates/src.conf.j2 +++ b/ansible/roles/build/templates/src.conf.j2 @@ -6,6 +6,8 @@ WITH_MALLOC_PRODUCTION=YES WITHOUT_LLVM_ASSERTIONS=YES WITH_REPRODUCIBLE_BUILD=YES PORTS_MODULES+=graphics/drm-510-kmod +PORTS_MODULES+=graphics/gpu-firmware-intel-kmod +PORTS_MODULES+=net/wireguard-kmod NO_SHARED=YES # Would be fun to experiment with: From fccb2312da69f3723b7649b3e791e26eb514bad4 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 1 Sep 2023 16:12:02 -0400 Subject: [PATCH 03/17] Start an ansible role for running a custom freebsd-update server. --- .../freebsd_update_server/tasks/common.yaml | 15 ++++++++++ .../freebsd_update_server/tasks/freebsd.yaml | 5 ++++ .../freebsd_update_server/tasks/linux.yaml | 29 +++++++++++++++++++ .../freebsd_update_server/tasks/main.yaml | 2 ++ .../freebsd_update_server/tasks/peruser.yaml | 29 +++++++++++++++++++ .../tasks/peruser_freebsd.yaml | 0 .../tasks/peruser_linux.yaml | 0 7 files changed, 80 insertions(+) create mode 100644 ansible/roles/freebsd_update_server/tasks/common.yaml create mode 100644 ansible/roles/freebsd_update_server/tasks/freebsd.yaml create mode 100644 ansible/roles/freebsd_update_server/tasks/linux.yaml create mode 100644 ansible/roles/freebsd_update_server/tasks/main.yaml create mode 100644 ansible/roles/freebsd_update_server/tasks/peruser.yaml create mode 100644 ansible/roles/freebsd_update_server/tasks/peruser_freebsd.yaml create mode 100644 ansible/roles/freebsd_update_server/tasks/peruser_linux.yaml diff --git a/ansible/roles/freebsd_update_server/tasks/common.yaml b/ansible/roles/freebsd_update_server/tasks/common.yaml new file mode 100644 index 0000000..fef1101 --- /dev/null +++ b/ansible/roles/freebsd_update_server/tasks/common.yaml @@ -0,0 +1,15 @@ +- import_tasks: tasks/freebsd.yaml + when: 'os_flavor == "freebsd"' + +- import_tasks: tasks/linux.yaml + when: 'os_flavor == "linux"' + +- include_tasks: + file: tasks/peruser.yaml + apply: + become: yes + become_user: "{{ initialize_user }}" + when: users is defined + loop: "{{ users | dict2items | community.general.json_query('[?value.initialize==`true`].key') }}" + loop_control: + loop_var: initialize_user diff --git a/ansible/roles/freebsd_update_server/tasks/freebsd.yaml b/ansible/roles/freebsd_update_server/tasks/freebsd.yaml new file mode 100644 index 0000000..b417174 --- /dev/null +++ b/ansible/roles/freebsd_update_server/tasks/freebsd.yaml @@ -0,0 +1,5 @@ +# - name: Install packages +# package: +# name: +# - foo +# state: present diff --git a/ansible/roles/freebsd_update_server/tasks/linux.yaml b/ansible/roles/freebsd_update_server/tasks/linux.yaml new file mode 100644 index 0000000..43ba876 --- /dev/null +++ b/ansible/roles/freebsd_update_server/tasks/linux.yaml @@ -0,0 +1,29 @@ +# - name: Build aur packages +# register: buildaur +# become_user: "{{ build_user.name }}" +# command: "aurutils-sync --no-view {{ item }}" +# args: +# creates: "/var/cache/pacman/custom/{{ item }}-*.pkg.tar.*" +# loop: +# - foo + +# - name: Update cache +# when: buildaur.changed +# pacman: +# name: [] +# state: present +# update_cache: true + +# - name: Install packages +# package: +# name: +# - foo +# state: present + +# - name: Enable services +# systemd: +# enabled: yes +# name: "{{ item }}" +# daemon_reload: yes +# loop: +# - foo.service diff --git a/ansible/roles/freebsd_update_server/tasks/main.yaml b/ansible/roles/freebsd_update_server/tasks/main.yaml new file mode 100644 index 0000000..6805b9d --- /dev/null +++ b/ansible/roles/freebsd_update_server/tasks/main.yaml @@ -0,0 +1,2 @@ +- import_tasks: tasks/common.yaml + # when: foo is defined diff --git a/ansible/roles/freebsd_update_server/tasks/peruser.yaml b/ansible/roles/freebsd_update_server/tasks/peruser.yaml new file mode 100644 index 0000000..111e886 --- /dev/null +++ b/ansible/roles/freebsd_update_server/tasks/peruser.yaml @@ -0,0 +1,29 @@ +- include_role: + name: per_user + +# - name: Create directories +# file: +# name: "{{ account_homedir.stdout }}/{{ item }}" +# state: directory +# mode: 0700 +# owner: "{{ account_name.stdout }}" +# group: "{{ group_name.stdout }}" +# loop: +# - ".config/foo" + +# - name: Copy files +# copy: +# src: "files/{{ item.src }}" +# dest: "{{ account_homedir.stdout }}/{{ item.dest }}" +# mode: 0600 +# owner: "{{ account_name.stdout }}" +# group: "{{ group_name.stdout }}" +# loop: +# - src: foo.conf +# dest: .config/foo/foo.conf + +- import_tasks: tasks/peruser_freebsd.yaml + when: 'os_flavor == "freebsd"' + +- import_tasks: tasks/peruser_linux.yaml + when: 'os_flavor == "linux"' diff --git a/ansible/roles/freebsd_update_server/tasks/peruser_freebsd.yaml b/ansible/roles/freebsd_update_server/tasks/peruser_freebsd.yaml new file mode 100644 index 0000000..e69de29 diff --git a/ansible/roles/freebsd_update_server/tasks/peruser_linux.yaml b/ansible/roles/freebsd_update_server/tasks/peruser_linux.yaml new file mode 100644 index 0000000..e69de29 From 7ecf2ef1b1fe5a43e3447e7ab93865c8d615ce7b Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 1 Sep 2023 16:41:34 -0400 Subject: [PATCH 04/17] Check out the freebsd-update-build git repo. --- ansible/playbook.yaml | 1 + .../freebsd_update_server/tasks/freebsd.yaml | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml index b826a66..94469fa 100644 --- a/ansible/playbook.yaml +++ b/ansible/playbook.yaml @@ -68,6 +68,7 @@ - portshaker - poudriere - poudrierenginx + - freebsd_update_server - hosts: mrmanager vars: diff --git a/ansible/roles/freebsd_update_server/tasks/freebsd.yaml b/ansible/roles/freebsd_update_server/tasks/freebsd.yaml index b417174..b2e1ac8 100644 --- a/ansible/roles/freebsd_update_server/tasks/freebsd.yaml +++ b/ansible/roles/freebsd_update_server/tasks/freebsd.yaml @@ -3,3 +3,23 @@ # name: # - foo # state: present + +- name: Create directories + file: + name: "{{ item }}" + state: directory + mode: 0755 + owner: "{{ build_user.name }}" + group: "{{ build_user.group }}" + loop: + - /opt/freebsd_update_server + +- name: Clone freebsd-update-build + git: + repo: "https://github.com/freebsd/freebsd-update-build.git" + dest: /opt/freebsd_update_server/freebsd-update-build + version: "28bb3ae7de9c1332fe8a366fb154a5b9faf37f49" + force: true + become: true + become_user: "{{ build_user.name }}" + diff: false From d20ce1e4d4b2fa8bc69bb7f4bb1fefbcfa14db32 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 1 Sep 2023 16:42:31 -0400 Subject: [PATCH 05/17] TEMP: Add the freebsd_update_server role to odofreebsd for development. --- ansible/playbook.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml index 94469fa..29d3d1f 100644 --- a/ansible/playbook.yaml +++ b/ansible/playbook.yaml @@ -119,3 +119,9 @@ ansible_become: True roles: - framework_laptop + +- hosts: odofreebsd + vars: + ansible_become: True + roles: + - freebsd_update_server \ No newline at end of file From 44d3cc61f5d605482391784ea57d5899e8f1a0df Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 1 Sep 2023 18:14:12 -0400 Subject: [PATCH 06/17] Add release.conf. --- ansible/roles/blank/tasks/common.yaml | 8 ++ .../files/build_release.bash | 15 +++ .../freebsd_update_server/files/release.conf | 118 ++++++++++++++++++ .../freebsd_update_server/tasks/common.yaml | 2 +- .../freebsd_update_server/tasks/freebsd.yaml | 22 ++++ 5 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 ansible/roles/freebsd_update_server/files/build_release.bash create mode 100644 ansible/roles/freebsd_update_server/files/release.conf diff --git a/ansible/roles/blank/tasks/common.yaml b/ansible/roles/blank/tasks/common.yaml index 386bbf1..bef243a 100644 --- a/ansible/roles/blank/tasks/common.yaml +++ b/ansible/roles/blank/tasks/common.yaml @@ -30,6 +30,14 @@ # - src: foo.conf # dest: /usr/local/etc/foo.conf +# - name: Clone Source +# git: +# repo: "https://foo.bar/baz.git" +# dest: /foo/bar +# version: "v1.0.2" +# force: true +# diff: false + - import_tasks: tasks/freebsd.yaml when: 'os_flavor == "freebsd"' diff --git a/ansible/roles/freebsd_update_server/files/build_release.bash b/ansible/roles/freebsd_update_server/files/build_release.bash new file mode 100644 index 0000000..9dc6185 --- /dev/null +++ b/ansible/roles/freebsd_update_server/files/build_release.bash @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +function main { + build_release +} + +function build_release { + /usr/src/release/release.sh -c /opt/freebsd_update_server/release.conf +} + +main "${@}" diff --git a/ansible/roles/freebsd_update_server/files/release.conf b/ansible/roles/freebsd_update_server/files/release.conf new file mode 100644 index 0000000..915bc0f --- /dev/null +++ b/ansible/roles/freebsd_update_server/files/release.conf @@ -0,0 +1,118 @@ +#!/bin/sh +# + +## Redefine environment variables here to override prototypes +## defined in release.sh. +#load_chroot_env() { } +#load_target_env() { } +#buildenv_setup() { } + +## Set the directory within which the release will be built. +CHROOTDIR="/scratch" + +## Do not explicitly require the devel/git port to be installed. +#NOGIT=1 +## Set the version control system host. +GITROOT="https://git.freebsd.org/" +GITSRC="src.git" +GITPORTS="ports.git" + +## Set the src/, ports/, and doc/ branches or tags. +#SRCBRANCH="stable/13" +SRCBRANCH="main" +PORTBRANCH="main" + +## Sample configuration for using git from ports. +#GITCMD="/usr/local/bin/git clone -q --branch main" + +## Set to override the default target architecture. +#TARGET="amd64" +#TARGET_ARCH="amd64" +#KERNEL="GENERIC" +KERNEL="GENERIC-NODEBUG" +## Multiple kernels may be set. +#KERNEL="GENERIC XENHVM" + +## Set to specify a custom make.conf and/or src.conf +#MAKE_CONF="/etc/local/make.conf" +#SRC_CONF="/etc/local/src.conf" + +## Set to use make(1) flags. +#MAKE_FLAGS="-s" + +## Set to use world- and kernel-specific make(1) flags. +#WORLD_FLAGS="-j $(sysctl -n hw.ncpu)" +#KERNEL_FLAGS="-j $(( $(( $(sysctl -n hw.ncpu) + 1 )) / 2 ))" + +## Set miscellaneous 'make release' settings. +#NOPORTS= +#NOSRC= +#WITH_DVD= +#WITH_COMPRESSED_IMAGES= + +## Set to '1' to disable multi-threaded xz(1) compression. +#XZ_THREADS=0 + +## Set when building embedded images. +#EMBEDDEDBUILD= + +## Set to a list of ports required to build embedded system-on-chip +## images, such as sysutils/u-boot-rpi. +#EMBEDDEDPORTS= + +## Set to the hardware platform of the target userland. This value +## is passed to make(1) to set the TARGET (value of uname -m) to cross +## build. +#EMBEDDED_TARGET= + +## Set to the machine processor architecture of the target userland. +## This value is passed to make(1) to set the TARGET_ARCH (value of uname -p) +## to cross build. +#EMBEDDED_TARGET_ARCH= + +## Set to skip the chroot environment buildworld/installworld/distribution +## step if it is expected the build environment will exist via alternate +## means. +#CHROOTBUILD_SKIP= + +## Set to a non-empty value skip checkout or update of /usr/src in +## the chroot. This is intended for use when /usr/src already exists. +#SRC_UPDATE_SKIP= + +## Set to a non-empty value skip checkout or update of /usr/ports in +## the chroot. This is intended for use when /usr/ports already exists. +#PORTS_UPDATE_SKIP= + +## Set to pass additional flags to make(1) for the build chroot setup, such +## as TARGET/TARGET_ARCH. +#CHROOT_MAKEENV= + +## Set to a non-empty value to build virtual machine images as part of the +## release build. +#WITH_VMIMAGES= + +## Set to a non-empty value to compress virtual machine images with xz(1) +## as part of the release build. +#WITH_COMPRESSED_VMIMAGES= + +## If WITH_VMIMAGES is set to a non-empty value, this is the name of the +## file to use for the installed userland/kernel. +#VMBASE="vm" + +## If WITH_VMIMAGES is set to a non-empty value, this is the size of the +## virtual machine disk filesystem. Valid size values are described in +## the makefs(8) manual page. +#VMSIZE="20g" + +## If WITH_VMIMAGES is set to a non-empty value, this is a list of disk +## image formats to create. Valid values are listed in the mkimg(1) +## manual page, as well as 'mkimg --formats' output. +#VMFORMATS="vhdf vmdk qcow2 raw" + +## Set to a non-empty value to build virtual machine images for various +## cloud providers as part of the release build. +#WITH_CLOUDWARE= + +## If WITH_CLOUDWARE is set to a non-empty value, this is a list of providers +## to create disk images. +#CLOUDWARE="EC2 GCE VAGRANT-VIRTUALBOX VAGRANT-VMWARE" diff --git a/ansible/roles/freebsd_update_server/tasks/common.yaml b/ansible/roles/freebsd_update_server/tasks/common.yaml index fef1101..764e0c6 100644 --- a/ansible/roles/freebsd_update_server/tasks/common.yaml +++ b/ansible/roles/freebsd_update_server/tasks/common.yaml @@ -1,5 +1,5 @@ - import_tasks: tasks/freebsd.yaml - when: 'os_flavor == "freebsd"' + when: 'os_flavor == "freebsd" and build_user is defined' - import_tasks: tasks/linux.yaml when: 'os_flavor == "linux"' diff --git a/ansible/roles/freebsd_update_server/tasks/freebsd.yaml b/ansible/roles/freebsd_update_server/tasks/freebsd.yaml index b2e1ac8..f7edb61 100644 --- a/ansible/roles/freebsd_update_server/tasks/freebsd.yaml +++ b/ansible/roles/freebsd_update_server/tasks/freebsd.yaml @@ -23,3 +23,25 @@ become: true become_user: "{{ build_user.name }}" diff: false + +- name: Install Configuration + copy: + src: "files/{{ item.src }}" + dest: "{{ item.dest }}" + mode: 0600 + owner: "{{ build_user.name }}" + group: "{{ build_user.group }}" + loop: + - src: release.conf + dest: /opt/freebsd_update_server/release.conf + +- name: Install scripts + copy: + src: "files/{{ item.src }}" + dest: "{{ item.dest }}" + mode: 0755 + owner: root + group: wheel + loop: + - src: build_release.bash + dest: /usr/local/bin/build_release From 6be5ad6b3d343b8a0f3f72caf66a45fa614ffa53 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 1 Sep 2023 18:51:23 -0400 Subject: [PATCH 07/17] Add a manual implementation of building a release. --- .../freebsd_update_server/files/build_release.bash | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ansible/roles/freebsd_update_server/files/build_release.bash b/ansible/roles/freebsd_update_server/files/build_release.bash index 9dc6185..90c53ac 100644 --- a/ansible/roles/freebsd_update_server/files/build_release.bash +++ b/ansible/roles/freebsd_update_server/files/build_release.bash @@ -5,11 +5,18 @@ IFS=$'\n\t' DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" function main { - build_release + build_release_script } -function build_release { +function build_release_script { /usr/src/release/release.sh -c /opt/freebsd_update_server/release.conf } +function build_release_manually { + make -C /usr/src buildworld buildkernel + make -C /usr/src/release obj + make -C /usr/src/release release + make -C /usr/src/release install DESTDIR=/where/to/put/the/images +} + main "${@}" From e7328f2865e646eb3fe5b832a1cca9dabea4faaf Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 4 Sep 2023 11:03:13 -0400 Subject: [PATCH 08/17] Use /etc/make.conf and /etc/src.conf. --- ansible/roles/freebsd_update_server/files/release.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/freebsd_update_server/files/release.conf b/ansible/roles/freebsd_update_server/files/release.conf index 915bc0f..f03758d 100644 --- a/ansible/roles/freebsd_update_server/files/release.conf +++ b/ansible/roles/freebsd_update_server/files/release.conf @@ -35,7 +35,9 @@ KERNEL="GENERIC-NODEBUG" ## Set to specify a custom make.conf and/or src.conf #MAKE_CONF="/etc/local/make.conf" +MAKE_CONF="/etc/make.conf" #SRC_CONF="/etc/local/src.conf" +SRC_CONF="/etc/src.conf" ## Set to use make(1) flags. #MAKE_FLAGS="-s" From 575d4360f1d8821d7dbeb041ee46cf84c7ffe0fe Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 5 Sep 2023 20:42:54 -0400 Subject: [PATCH 09/17] Build current instead of 13.2. --- ansible/roles/build/defaults/main.yaml | 3 ++- ansible/roles/build/templates/src.conf.j2 | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/build/defaults/main.yaml b/ansible/roles/build/defaults/main.yaml index cb87021..b91021e 100644 --- a/ansible/roles/build/defaults/main.yaml +++ b/ansible/roles/build/defaults/main.yaml @@ -1 +1,2 @@ -freebsd_version: "releng/13.2" +# freebsd_version: "releng/13.2" +freebsd_version: "815b7436a7c6302365b6514194d27d41cb736227" diff --git a/ansible/roles/build/templates/src.conf.j2 b/ansible/roles/build/templates/src.conf.j2 index 75f0dec..1311334 100644 --- a/ansible/roles/build/templates/src.conf.j2 +++ b/ansible/roles/build/templates/src.conf.j2 @@ -8,7 +8,6 @@ WITH_REPRODUCIBLE_BUILD=YES PORTS_MODULES+=graphics/drm-510-kmod PORTS_MODULES+=graphics/gpu-firmware-intel-kmod PORTS_MODULES+=net/wireguard-kmod -NO_SHARED=YES # Would be fun to experiment with: # WITHOUT_SOURCELESS=YES From d56132618afef398c40b8b351823a7c50c8a40fa Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Tue, 5 Sep 2023 21:25:23 -0400 Subject: [PATCH 10/17] Enable guc for hw accelerated encoding for wl-screenrec. --- ansible/roles/graphics/files/intel_hw_accel_video_loader.conf | 2 ++ ansible/roles/graphics/tasks/freebsd_intel.yaml | 1 + 2 files changed, 3 insertions(+) create mode 100644 ansible/roles/graphics/files/intel_hw_accel_video_loader.conf diff --git a/ansible/roles/graphics/files/intel_hw_accel_video_loader.conf b/ansible/roles/graphics/files/intel_hw_accel_video_loader.conf new file mode 100644 index 0000000..06840b2 --- /dev/null +++ b/ansible/roles/graphics/files/intel_hw_accel_video_loader.conf @@ -0,0 +1,2 @@ +#enable_guc=2 +hw.i915kms.enable_guc=2 diff --git a/ansible/roles/graphics/tasks/freebsd_intel.yaml b/ansible/roles/graphics/tasks/freebsd_intel.yaml index 96a1c82..1385e24 100644 --- a/ansible/roles/graphics/tasks/freebsd_intel.yaml +++ b/ansible/roles/graphics/tasks/freebsd_intel.yaml @@ -29,6 +29,7 @@ group: wheel loop: - intel_power + - intel_hw_accel_video - name: Install service configuration copy: From 1c922c2234062ebeb45ec057e5267007afb9f9dc Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Thu, 7 Sep 2023 16:13:32 -0400 Subject: [PATCH 11/17] Set up a separate FreeBSD update VM. It seems to compile the ports kernel modules into the OS, I need to be running the same kernel version as I am building, so I am putting it into its own VM. --- .../vm/host_vars/freebsdupdatemrmanager | 4 +++ ansible/environments/vm/hosts | 4 +++ ansible/playbook.yaml | 12 +++++++- ansible/roles/build/meta/main.yaml | 6 ++-- ansible/roles/build/tasks/common.yaml | 9 ------ ansible/roles/build/tasks/peruser.yaml | 29 ------------------- .../roles/build/tasks/peruser_freebsd.yaml | 0 ansible/roles/build/tasks/peruser_linux.yaml | 0 .../freebsd_update_server/tasks/common.yaml | 10 ------- .../freebsd_update_server/tasks/freebsd.yaml | 11 +++---- .../freebsd_update_server/tasks/peruser.yaml | 29 ------------------- .../tasks/peruser_freebsd.yaml | 0 .../tasks/peruser_linux.yaml | 0 ansible/roles/fstab/tasks/common.yaml | 10 ------- ansible/roles/fstab/tasks/peruser.yaml | 29 ------------------- .../roles/fstab/tasks/peruser_freebsd.yaml | 0 ansible/roles/fstab/tasks/peruser_linux.yaml | 0 .../roles/jail_nat_dhcp/files/kea-dhcp4.conf | 5 ++++ ansible/run.bash | 2 ++ 19 files changed, 36 insertions(+), 124 deletions(-) create mode 100644 ansible/environments/vm/host_vars/freebsdupdatemrmanager delete mode 100644 ansible/roles/build/tasks/peruser.yaml delete mode 100644 ansible/roles/build/tasks/peruser_freebsd.yaml delete mode 100644 ansible/roles/build/tasks/peruser_linux.yaml delete mode 100644 ansible/roles/freebsd_update_server/tasks/peruser.yaml delete mode 100644 ansible/roles/freebsd_update_server/tasks/peruser_freebsd.yaml delete mode 100644 ansible/roles/freebsd_update_server/tasks/peruser_linux.yaml delete mode 100644 ansible/roles/fstab/tasks/peruser.yaml delete mode 100644 ansible/roles/fstab/tasks/peruser_freebsd.yaml delete mode 100644 ansible/roles/fstab/tasks/peruser_linux.yaml diff --git a/ansible/environments/vm/host_vars/freebsdupdatemrmanager b/ansible/environments/vm/host_vars/freebsdupdatemrmanager new file mode 100644 index 0000000..ef13840 --- /dev/null +++ b/ansible/environments/vm/host_vars/freebsdupdatemrmanager @@ -0,0 +1,4 @@ +os_flavor: "freebsd" +build_user: + name: root + group: wheel diff --git a/ansible/environments/vm/hosts b/ansible/environments/vm/hosts index afaa022..efffe2b 100644 --- a/ansible/environments/vm/hosts +++ b/ansible/environments/vm/hosts @@ -1,9 +1,13 @@ [vm] poudriereodo ansible_user=builder ansible_host=10.213.177.12 poudrieremrmanager ansible_user=root ansible_host=poudriere +freebsdupdatemrmanager ansible_user=root ansible_host=freebsdupdate # # Put in ~/.ssh/config # Host poudriere # ProxyJump talexander@mrmanager # HostName 10.215.1.203 # +# Host freebsdupdate +# ProxyJump talexander@mrmanager +# HostName 10.215.1.213 diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml index 29d3d1f..dcc90d4 100644 --- a/ansible/playbook.yaml +++ b/ansible/playbook.yaml @@ -124,4 +124,14 @@ vars: ansible_become: True roles: - - freebsd_update_server \ No newline at end of file + - freebsd_update_server + +- hosts: freebsdupdatemrmanager + vars: + ansible_become: True + roles: + - sudo # for poudboot script + - doas + - fstab + - build + - freebsd_update_server \ No newline at end of file diff --git a/ansible/roles/build/meta/main.yaml b/ansible/roles/build/meta/main.yaml index 8586ba4..d62eaf8 100644 --- a/ansible/roles/build/meta/main.yaml +++ b/ansible/roles/build/meta/main.yaml @@ -1,3 +1,5 @@ dependencies: - - users - - gpg + - role: users + when: 'os_flavor == "linux"' + - role: gpg + when: 'os_flavor == "linux"' diff --git a/ansible/roles/build/tasks/common.yaml b/ansible/roles/build/tasks/common.yaml index 225b20c..b0a9a0a 100644 --- a/ansible/roles/build/tasks/common.yaml +++ b/ansible/roles/build/tasks/common.yaml @@ -3,12 +3,3 @@ - import_tasks: tasks/linux.yaml when: 'os_flavor == "linux"' - -- include_tasks: - file: tasks/peruser.yaml - apply: - become: yes - become_user: "{{ initialize_user }}" - loop: "{{ users | dict2items | community.general.json_query('[?value.initialize==`true`].key') }}" - loop_control: - loop_var: initialize_user diff --git a/ansible/roles/build/tasks/peruser.yaml b/ansible/roles/build/tasks/peruser.yaml deleted file mode 100644 index 111e886..0000000 --- a/ansible/roles/build/tasks/peruser.yaml +++ /dev/null @@ -1,29 +0,0 @@ -- include_role: - name: per_user - -# - name: Create directories -# file: -# name: "{{ account_homedir.stdout }}/{{ item }}" -# state: directory -# mode: 0700 -# owner: "{{ account_name.stdout }}" -# group: "{{ group_name.stdout }}" -# loop: -# - ".config/foo" - -# - name: Copy files -# copy: -# src: "files/{{ item.src }}" -# dest: "{{ account_homedir.stdout }}/{{ item.dest }}" -# mode: 0600 -# owner: "{{ account_name.stdout }}" -# group: "{{ group_name.stdout }}" -# loop: -# - src: foo.conf -# dest: .config/foo/foo.conf - -- import_tasks: tasks/peruser_freebsd.yaml - when: 'os_flavor == "freebsd"' - -- import_tasks: tasks/peruser_linux.yaml - when: 'os_flavor == "linux"' diff --git a/ansible/roles/build/tasks/peruser_freebsd.yaml b/ansible/roles/build/tasks/peruser_freebsd.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/ansible/roles/build/tasks/peruser_linux.yaml b/ansible/roles/build/tasks/peruser_linux.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/ansible/roles/freebsd_update_server/tasks/common.yaml b/ansible/roles/freebsd_update_server/tasks/common.yaml index 764e0c6..b0a9a0a 100644 --- a/ansible/roles/freebsd_update_server/tasks/common.yaml +++ b/ansible/roles/freebsd_update_server/tasks/common.yaml @@ -3,13 +3,3 @@ - import_tasks: tasks/linux.yaml when: 'os_flavor == "linux"' - -- include_tasks: - file: tasks/peruser.yaml - apply: - become: yes - become_user: "{{ initialize_user }}" - when: users is defined - loop: "{{ users | dict2items | community.general.json_query('[?value.initialize==`true`].key') }}" - loop_control: - loop_var: initialize_user diff --git a/ansible/roles/freebsd_update_server/tasks/freebsd.yaml b/ansible/roles/freebsd_update_server/tasks/freebsd.yaml index f7edb61..e5139a0 100644 --- a/ansible/roles/freebsd_update_server/tasks/freebsd.yaml +++ b/ansible/roles/freebsd_update_server/tasks/freebsd.yaml @@ -1,8 +1,9 @@ -# - name: Install packages -# package: -# name: -# - foo -# state: present +- name: Install packages + package: + name: + - git + - tmux # For convenience + state: present - name: Create directories file: diff --git a/ansible/roles/freebsd_update_server/tasks/peruser.yaml b/ansible/roles/freebsd_update_server/tasks/peruser.yaml deleted file mode 100644 index 111e886..0000000 --- a/ansible/roles/freebsd_update_server/tasks/peruser.yaml +++ /dev/null @@ -1,29 +0,0 @@ -- include_role: - name: per_user - -# - name: Create directories -# file: -# name: "{{ account_homedir.stdout }}/{{ item }}" -# state: directory -# mode: 0700 -# owner: "{{ account_name.stdout }}" -# group: "{{ group_name.stdout }}" -# loop: -# - ".config/foo" - -# - name: Copy files -# copy: -# src: "files/{{ item.src }}" -# dest: "{{ account_homedir.stdout }}/{{ item.dest }}" -# mode: 0600 -# owner: "{{ account_name.stdout }}" -# group: "{{ group_name.stdout }}" -# loop: -# - src: foo.conf -# dest: .config/foo/foo.conf - -- import_tasks: tasks/peruser_freebsd.yaml - when: 'os_flavor == "freebsd"' - -- import_tasks: tasks/peruser_linux.yaml - when: 'os_flavor == "linux"' diff --git a/ansible/roles/freebsd_update_server/tasks/peruser_freebsd.yaml b/ansible/roles/freebsd_update_server/tasks/peruser_freebsd.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/ansible/roles/freebsd_update_server/tasks/peruser_linux.yaml b/ansible/roles/freebsd_update_server/tasks/peruser_linux.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/ansible/roles/fstab/tasks/common.yaml b/ansible/roles/fstab/tasks/common.yaml index fef1101..c4f2d20 100644 --- a/ansible/roles/fstab/tasks/common.yaml +++ b/ansible/roles/fstab/tasks/common.yaml @@ -3,13 +3,3 @@ - import_tasks: tasks/linux.yaml when: 'os_flavor == "linux"' - -- include_tasks: - file: tasks/peruser.yaml - apply: - become: yes - become_user: "{{ initialize_user }}" - when: users is defined - loop: "{{ users | dict2items | community.general.json_query('[?value.initialize==`true`].key') }}" - loop_control: - loop_var: initialize_user diff --git a/ansible/roles/fstab/tasks/peruser.yaml b/ansible/roles/fstab/tasks/peruser.yaml deleted file mode 100644 index 111e886..0000000 --- a/ansible/roles/fstab/tasks/peruser.yaml +++ /dev/null @@ -1,29 +0,0 @@ -- include_role: - name: per_user - -# - name: Create directories -# file: -# name: "{{ account_homedir.stdout }}/{{ item }}" -# state: directory -# mode: 0700 -# owner: "{{ account_name.stdout }}" -# group: "{{ group_name.stdout }}" -# loop: -# - ".config/foo" - -# - name: Copy files -# copy: -# src: "files/{{ item.src }}" -# dest: "{{ account_homedir.stdout }}/{{ item.dest }}" -# mode: 0600 -# owner: "{{ account_name.stdout }}" -# group: "{{ group_name.stdout }}" -# loop: -# - src: foo.conf -# dest: .config/foo/foo.conf - -- import_tasks: tasks/peruser_freebsd.yaml - when: 'os_flavor == "freebsd"' - -- import_tasks: tasks/peruser_linux.yaml - when: 'os_flavor == "linux"' diff --git a/ansible/roles/fstab/tasks/peruser_freebsd.yaml b/ansible/roles/fstab/tasks/peruser_freebsd.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/ansible/roles/fstab/tasks/peruser_linux.yaml b/ansible/roles/fstab/tasks/peruser_linux.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf b/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf index f02f4b3..54a4996 100644 --- a/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf +++ b/ansible/roles/jail_nat_dhcp/files/kea-dhcp4.conf @@ -78,6 +78,11 @@ // brianai "hw-address": "06:a6:dc:59:78:12", "ip-address": "10.215.1.215" + }, + { + // freebsdupdate + "hw-address": "06:14:5c:92:3d:5b", + "ip-address": "10.215.1.213" } ] } diff --git a/ansible/run.bash b/ansible/run.bash index 294ba62..7021801 100755 --- a/ansible/run.bash +++ b/ansible/run.bash @@ -38,6 +38,8 @@ elif [ "$target" = "jail_admin_git" ]; then ansible-playbook -v -i environments/jail playbook.yaml --diff --limit admin_git "${@}" elif [ "$target" = "jail_public_dns" ]; then ansible-playbook -v -i environments/jail playbook.yaml --diff --limit public_dns "${@}" +elif [ "$target" = "vm_freebsdupdatemrmanager" ]; then + ansible-playbook -v -i environments/vm playbook.yaml --diff --limit freebsdupdatemrmanager "${@}" else die 1 "Unrecognized target" fi From da36f1b3d8ae77d4b39c745b069863408c5cdf41 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 8 Sep 2023 10:51:12 -0400 Subject: [PATCH 12/17] Update build_release script to update the virtual machine OS. --- ansible/roles/build/templates/src.conf.j2 | 2 +- .../files/build_release.bash | 118 +++++++++++++++++- .../freebsd_update_server/tasks/freebsd.yaml | 2 + 3 files changed, 116 insertions(+), 6 deletions(-) diff --git a/ansible/roles/build/templates/src.conf.j2 b/ansible/roles/build/templates/src.conf.j2 index 1311334..d4d2396 100644 --- a/ansible/roles/build/templates/src.conf.j2 +++ b/ansible/roles/build/templates/src.conf.j2 @@ -5,7 +5,7 @@ KERNCONF=CUSTOM WITH_MALLOC_PRODUCTION=YES WITHOUT_LLVM_ASSERTIONS=YES WITH_REPRODUCIBLE_BUILD=YES -PORTS_MODULES+=graphics/drm-510-kmod +PORTS_MODULES+=graphics/drm-kmod PORTS_MODULES+=graphics/gpu-firmware-intel-kmod PORTS_MODULES+=net/wireguard-kmod diff --git a/ansible/roles/freebsd_update_server/files/build_release.bash b/ansible/roles/freebsd_update_server/files/build_release.bash index 90c53ac..92cc305 100644 --- a/ansible/roles/freebsd_update_server/files/build_release.bash +++ b/ansible/roles/freebsd_update_server/files/build_release.bash @@ -4,19 +4,127 @@ set -euo pipefail IFS=$'\n\t' DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +: ${DATA_DIRECTORY:="/usr/local/share/freebsdupdate"} +: ${STAGE_FILE:="${DATA_DIRECTORY}/stage"} +: ${RELEASE_DIRECTORY:="${DATA_DIRECTORY}/release"} +: ${LOG_DIRECTORY:="${DATA_DIRECTORY}/logs"} +: ${PORTS_TREE:="/usr/ports"} +: ${PORTS_REPO:="https://git.FreeBSD.org/ports.git"} + +############## Setup ######################### + +function die { + local status_code="$1" + shift + (>&2 echo "${@}") + exit "$status_code" +} + +function log { + (>&2 echo "${@}") +} + +############## Program ######################### + function main { - build_release_script + assert_directories + + local stage="" + if [ -e "$STAGE_FILE" ]; then + local stage=$(cat "$STAGE_FILE") + fi + if [ "$stage" = "selfbuild" ]; then + log_cmd stage_selfbuild + elif [ "$stage" = "selfinstallworld" ]; then + log_cmd stage_selfinstallworld + elif [ "$stage" = "selfconflictcheck" ]; then + log_cmd stage_selfconflictcheck + elif [ "$stage" = "releasebuild" ]; then + log_cmd stage_releasebuild + elif [ "$stage" = "done" ]; then + log_cmd stage_done + else + die 1 "Unhandled stage: \"$stage\"." + fi } -function build_release_script { - /usr/src/release/release.sh -c /opt/freebsd_update_server/release.conf +function log_cmd { + "${@}" |& tee "$LOG_DIRECTORY/$(date +%Y%m%d-%s).log" } -function build_release_manually { +function self_conflict_check { + if etcupdate status | grep -qE '^ C '; then + die 1 'Conflicts remain in etcupdate. Run `etcupdate resolve` to fix them first.' + fi +} + +function assert_directories { + for d in "$DATA_DIRECTORY" "$RELEASE_DIRECTORY" "$LOG_DIRECTORY"; do + if [ ! -e "$d" ]; then + mkdir -p "$d" + fi + done +} + +function update_ports_tree { + if [ ! -e "$PORTS_TREE" ]; then + mkdir -p $PORTS_TREE + git -C $PORTS_TREE init --initial-branch=main + git -C $PORTS_TREE remote add origin $PORTS_REPO + fi + git -C $PORTS_TREE fetch origin main # 'refs/heads/main' + git -C $PORTS_TREE checkout FETCH_HEAD +} + +function set_stage { + echo "${@}" > "$STAGE_FILE" +} + +function stage_selfbuild { + self_conflict_check + assert_directories + update_ports_tree + + SRCCONF=/dev/null __MAKE_CONF=/dev/null make -C /usr/src clean + SRCCONF=/dev/null __MAKE_CONF=/dev/null make -C /usr/src buildworld buildkernel + SRCCONF=/dev/null __MAKE_CONF=/dev/null make -C /usr/src installkernel + + set_stage "selfinstallworld" + /sbin/shutdown -r now +} + +function stage_selfinstallworld { + etcupdate -p + SRCCONF=/dev/null __MAKE_CONF=/dev/null make -C /usr/src installworld + etcupdate -B + + set_stage "selfconflictcheck" + stage_selfconflictcheck +} + +function stage_selfconflictcheck { + self_conflict_check + set_stage "releasebuild" + /sbin/shutdown -r now +} + +function stage_releasebuild { + local today=$(date +%Y%m%d) + local target_directory="${RELEASE_DIRECTORY}/${today}" + if [ -e "$target_directory" ]; then + die 1 "The release directory $target_directory already exists. Exiting." + fi + SRCCONF=/dev/null __MAKE_CONF=/dev/null make -C /usr/src clean make -C /usr/src buildworld buildkernel make -C /usr/src/release obj make -C /usr/src/release release - make -C /usr/src/release install DESTDIR=/where/to/put/the/images + mkdir -p "$target_directory" + make -C /usr/src/release install DESTDIR="$target_directory" + set_stage "done" +} + +function stage_done { + log "Everything is done." } main "${@}" diff --git a/ansible/roles/freebsd_update_server/tasks/freebsd.yaml b/ansible/roles/freebsd_update_server/tasks/freebsd.yaml index e5139a0..68ab5d4 100644 --- a/ansible/roles/freebsd_update_server/tasks/freebsd.yaml +++ b/ansible/roles/freebsd_update_server/tasks/freebsd.yaml @@ -3,6 +3,8 @@ name: - git - tmux # For convenience + - htop # For convenience + - bash state: present - name: Create directories From 34ffd5c100ade26df45a1da227b6f9d145dd32e1 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 8 Sep 2023 12:43:26 -0400 Subject: [PATCH 13/17] Build FreeBSD for tigerlake on freebsdupdate. I think I need to move to specifying separate src.conf files for each build but I am going to get this working in a simple setup first. --- ansible/environments/vm/host_vars/freebsdupdatemrmanager | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/environments/vm/host_vars/freebsdupdatemrmanager b/ansible/environments/vm/host_vars/freebsdupdatemrmanager index ef13840..f68e738 100644 --- a/ansible/environments/vm/host_vars/freebsdupdatemrmanager +++ b/ansible/environments/vm/host_vars/freebsdupdatemrmanager @@ -1,4 +1,5 @@ os_flavor: "freebsd" +cpu_opt: tigerlake build_user: name: root group: wheel From 72a8d6f6156d1a865cd929fed505bcdeff522bf6 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 8 Sep 2023 16:39:43 -0400 Subject: [PATCH 14/17] Build the ports modules with world instead of just with the kernel. I'm hoping this fixes an issue where the chroot for building the ports only contains the kernel files and therefore lacks /bin/sh. --- ansible/roles/build/templates/src.conf.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/build/templates/src.conf.j2 b/ansible/roles/build/templates/src.conf.j2 index d4d2396..9a71c0c 100644 --- a/ansible/roles/build/templates/src.conf.j2 +++ b/ansible/roles/build/templates/src.conf.j2 @@ -5,6 +5,7 @@ KERNCONF=CUSTOM WITH_MALLOC_PRODUCTION=YES WITHOUT_LLVM_ASSERTIONS=YES WITH_REPRODUCIBLE_BUILD=YES +MODULES_WITH_WORLD=YES PORTS_MODULES+=graphics/drm-kmod PORTS_MODULES+=graphics/gpu-firmware-intel-kmod PORTS_MODULES+=net/wireguard-kmod From 2d260dec905fe730bb985f8bdf76a88d84ae6f5e Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 8 Sep 2023 23:36:59 -0400 Subject: [PATCH 15/17] Update the FreeBSD version. --- ansible/roles/build/defaults/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/build/defaults/main.yaml b/ansible/roles/build/defaults/main.yaml index b91021e..dbfe087 100644 --- a/ansible/roles/build/defaults/main.yaml +++ b/ansible/roles/build/defaults/main.yaml @@ -1,2 +1,2 @@ # freebsd_version: "releng/13.2" -freebsd_version: "815b7436a7c6302365b6514194d27d41cb736227" +freebsd_version: "9c80d66ec1b4c5b9ac7aaf5b0fdbb1628d49c181" From 51e5917e4334cc6de73a8f5bc4204592ac69c8bd Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 16 Sep 2023 21:29:30 -0400 Subject: [PATCH 16/17] Remove MODULES_WITH_WORLD. --- ansible/playbook.yaml | 4 ++-- ansible/roles/build/templates/src.conf.j2 | 1 - ansible/roles/graphics/files/intel_hw_accel_video_loader.conf | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml index dcc90d4..7e7bfb3 100644 --- a/ansible/playbook.yaml +++ b/ansible/playbook.yaml @@ -124,7 +124,7 @@ vars: ansible_become: True roles: - - freebsd_update_server + - freebsd_update_server - hosts: freebsdupdatemrmanager vars: @@ -134,4 +134,4 @@ - doas - fstab - build - - freebsd_update_server \ No newline at end of file + - freebsd_update_server diff --git a/ansible/roles/build/templates/src.conf.j2 b/ansible/roles/build/templates/src.conf.j2 index 9a71c0c..d4d2396 100644 --- a/ansible/roles/build/templates/src.conf.j2 +++ b/ansible/roles/build/templates/src.conf.j2 @@ -5,7 +5,6 @@ KERNCONF=CUSTOM WITH_MALLOC_PRODUCTION=YES WITHOUT_LLVM_ASSERTIONS=YES WITH_REPRODUCIBLE_BUILD=YES -MODULES_WITH_WORLD=YES PORTS_MODULES+=graphics/drm-kmod PORTS_MODULES+=graphics/gpu-firmware-intel-kmod PORTS_MODULES+=net/wireguard-kmod diff --git a/ansible/roles/graphics/files/intel_hw_accel_video_loader.conf b/ansible/roles/graphics/files/intel_hw_accel_video_loader.conf index 06840b2..64c4ca9 100644 --- a/ansible/roles/graphics/files/intel_hw_accel_video_loader.conf +++ b/ansible/roles/graphics/files/intel_hw_accel_video_loader.conf @@ -1,2 +1,2 @@ #enable_guc=2 -hw.i915kms.enable_guc=2 +#hw.i915kms.enable_guc=2 From e3e7de8eb19f7dd44c6ed3f5509bc2cc73bcb0b7 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 14 Oct 2023 14:30:34 -0400 Subject: [PATCH 17/17] The SHARED_TOOLCHAIN option was removed in FreeBSD 14. --- ansible/roles/build/templates/src.conf.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/roles/build/templates/src.conf.j2 b/ansible/roles/build/templates/src.conf.j2 index d4d2396..cbb2fe8 100644 --- a/ansible/roles/build/templates/src.conf.j2 +++ b/ansible/roles/build/templates/src.conf.j2 @@ -11,7 +11,6 @@ PORTS_MODULES+=net/wireguard-kmod # Would be fun to experiment with: # WITHOUT_SOURCELESS=YES -# WITHOUT_SHARED_TOOLCHAIN=YES # WITHOUT_GAMES=YES # WITHOUT_KERBEROS=YES # WITHOUT_LEGACY_CONSOLE=YES