mirror/freebsd-ports
1
0
Fork 0
mirror of https://git.FreeBSD.org/ports.git synced 2024-10-17 19:39:43 +00:00
Code Issues Releases Activity

lang/python: add bytecode trigger

Browse Source 
Facilitates compiling, writing and removing bytecode files (.pyc)
in site-packages after all pkg transactions have been completed.

Technical details: https://wiki.freebsd.org/Python/CompiledPackages

Fixes reports of Python port builds as root failing on filesystem
violations due to bytecode file writes where the port did not include
them in the package.

For those ports/packages that currently package bytecode, some
checksum mismatches on those files may occur. This is harmless and
will be rectified, in large as part of a USE_PYTHON=distutils
overhaul to reduce churn.

While here, implement a long-standing todo item of letting lang/python
ports use python.mk bits. Not only does this obviate duplicate
variables in each Makefile, but SUB_LIST (also added) is used for
these triggers.

Co-authored by: tcberner
Approved by: tcberner (mentor)
Differential Revision: https://reviews.freebsd.org/D34739
This commit is contained in:
Charlie Li 2023-02-15 15:57:06 -05:00
parent 9cdba749a8
commit c17ddfbf66
No known key found for this signature in database
GPG Key ID: FEB7852BE29B3E87
13 changed files with 269 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CHANGES
View File

@ -10,6 +10,14 @@ in the release notes and/or placed into UPDATING.
All ports committers are allowed to commit to this file.
20230215:
AUTHOR: vishwin@FreeBSD.org
USES=python now includes SUB_LIST entries to facilitate common
substitutions for python.mk variables other than in ${PLIST}.
They are the same as PLIST_SUB, except PYTHON_INCLUDEDIR,
PYTHON_LIBDIR and PYTHON_SITELIBDIR include ${PREFIX}.
20230111:
AUTHOR: vishwin@FreeBSD.org

25
Mk/Uses/python.mk
View File

@ -257,17 +257,20 @@
# packages for different Python versions.
# default: -py${PYTHON_SUFFIX}
#
# Using USES=python also will add some useful entries to PLIST_SUB:
# Using USES=python also will add some useful entries to SUB_LIST and PLIST_SUB:
#
# PYTHON_INCLUDEDIR=${PYTHONPREFIX_INCLUDEDIR:S;${PREFIX}/;;}
# PYTHON_LIBDIR=${PYTHONPREFIX_LIBDIR:S;${PREFIX}/;;}
# PYTHON_INCLUDEDIR=${PYTHONPREFIX_INCLUDEDIR}
# PYTHON_LIBDIR=${PYTHONPREFIX_LIBDIR}
# PYTHON_PLATFORM=${PYTHON_PLATFORM}
# PYTHON_SITELIBDIR=${PYTHONPREFIX_SITELIBDIR:S;${PREFIX}/;;}
# PYTHON_SITELIBDIR=${PYTHONPREFIX_SITELIBDIR}
# PYTHON_SUFFIX=${PYTHON_SUFFIX}
# PYTHON_VER=${PYTHON_VER}
# PYTHON_VERSION=${PYTHON_VERSION}
#
# and PYTHON2 and PYTHON3 will be set according to the Python version:
# where PYTHON_INCLUDEDIR, PYTHON_LIBDIR and PYTHON_SITELIBDIR have their PREFIX
# stripped for PLIST_SUB.
#
# PYTHON2 and PYTHON3 will also be set according to the Python version:
#
# PYTHON2="" PYTHON3="@comment " for Python 2.x
# PYTHON2="@comment " PYTHON3="" for Python 3.x
@ -785,6 +788,16 @@ ${_stage}_DEPENDS+= ${PYTHON_CMD}:${PYTHON_PORTSDIR}
PREFIX= ${PYTHONBASE}
. endif
# Substitutions for SUB_FILES
SUB_LIST+= PYTHON_INCLUDEDIR=${PYTHONPREFIX_INCLUDEDIR} \
PYTHON_LIBDIR=${PYTHONPREFIX_LIBDIR} \
PYTHON_PLATFORM=${PYTHON_PLATFORM} \
PYTHON_SITELIBDIR=${PYTHONPREFIX_SITELIBDIR} \
PYTHON_SUFFIX=${PYTHON_SUFFIX} \
PYTHON_EXT_SUFFIX=${PYTHON_EXT_SUFFIX} \
PYTHON_VER=${PYTHON_VER} \
PYTHON_VERSION=${PYTHON_VERSION}
# Substitutions for pkg-plist
# Use a short form of the PYTHONPREFIX_*DIR variables; we don't need the
# base directory in the plist file.
@ -797,8 +810,10 @@ PLIST_SUB+= PYTHON_INCLUDEDIR=${PYTHONPREFIX_INCLUDEDIR:S;${PREFIX}/;;} \
PYTHON_VER=${PYTHON_VER} \
PYTHON_VERSION=${PYTHON_VERSION}
. if ${PYTHON_REL} < 30000
SUB_LIST+= PYTHON2="" PYTHON3="@comment "
PLIST_SUB+= PYTHON2="" PYTHON3="@comment "
. else
SUB_LIST+= PYTHON2="@comment " PYTHON3=""
PLIST_SUB+= PYTHON2="@comment " PYTHON3=""
. endif

17
UPDATING
View File

@ -5,6 +5,23 @@ they are unavoidable.
You should get into the habit of checking this file for changes each time
you update your ports collection, before attempting any port upgrades.
20230215:
AFFECTS: users of python
AUTHOR: vishwin@FreeBSD.org
A trigger has been added to the lang/python3* ports to compile,
write and remove bytecode files (.pyc) in site-packages after all
pkg transactions have been completed. pkg will no longer manage
such files directly, as they are meant to be generated after
installation.
For those ports/packages that still package bytecode, some package
checksum mismatches on those files may occur. This is harmless
and will be rectified.
Technical details available at:
https://wiki.freebsd.org/Python/CompiledPackages
20230213:
Affects: users of sysutils/nut*
AUTHOR: cy@FreeBSD.org

11
lang/python310/Makefile
View File

@ -1,5 +1,6 @@
PORTNAME= python
DISTVERSION= ${PYTHON_DISTVERSION}
PORTREVISION= 1
CATEGORIES= lang python
MASTER_SITES= PYTHON/ftp/python/${DISTVERSION:C/[a-z].*//}
PKGNAMESUFFIX= ${PYTHON_SUFFIX}
@ -14,8 +15,8 @@ LICENSE= PSFL
LIB_DEPENDS= libffi.so:devel/libffi
USES= compiler:c11 cpe ncurses pathfix pkgconfig readline \
shebangfix ssl tar:xz
USES= compiler:c11 cpe ncurses pathfix pkgconfig \
python:${PYTHON_DISTVERSION:R},env readline shebangfix ssl tar:xz trigger
PATHFIX_MAKEFILEIN= Makefile.pre.in
USE_LDCONFIG= yes
GNU_CONFIGURE= yes
@ -24,11 +25,7 @@ SHEBANG_FILES= Lib/*.py Lib/*/*.py Lib/*/*/*.py Lib/*/*/*/*.py
SHEBANG_FILES+= Lib/test/ziptestdata/exe_with_z64 \
Lib/test/ziptestdata/exe_with_zip \
Lib/test/ziptestdata/header.sh
# Duplicate python.mk variables. TODO: Let lang/python?? ports use python.mk bits.
PYTHON_VER= ${PYTHON_DISTVERSION:R}
PYTHON_VERSION= python${PYTHON_VER}
PYTHON_SUFFIX= ${PYTHON_VER:S/.//g}
TRIGGERS= ${PYTHON_VERSION}
DISABLED_EXTENSIONS= _sqlite3 _tkinter _gdbm
CONFIGURE_ARGS+= --enable-shared --without-ensurepip \

40
lang/python310/files/python3.10.ucl.in Normal file
View File

@ -0,0 +1,40 @@
path_glob: "%%PYTHON_SITELIBDIR%%/*"
trigger: {
type: lua
sandbox: false
script: <<EOS
function cleanup(directory)
for _,d in ipairs(pkg.readdir(directory)) do
local full_path = directory .. "/" .. d
local stat = pkg.stat(full_path)
if stat["type"] == "dir" then
if (d ~= "__pycache__") then
cleanup(full_path)
else
for _,bytecode_file in ipairs(pkg.readdir(full_path)) do
local file_origin = string.gsub(bytecode_file, "[.]cpython[-]%%PYTHON_SUFFIX%%[.].*pyc", ".py")
if file_origin then
local origin_path = directory .. "/" .. file_origin
if (not pkg.stat(origin_path)) then
--print(" >=> removed stale bytecode " .. bytecode_file)
os.remove(full_path .. "/" .. bytecode_file)
end
end
end
end
local res = pkg.readdir(full_path)
if #res == 0 then
--print(" >=> removed empty directory " .. full_path )
os.remove(full_path)
end
end
end
end
print(">=> Cleaning stale bytecode files...")
cleanup("%%PYTHON_SITELIBDIR%%")
print(">=> Byte-compiling Python source files...")
pkg.exec({"%%PYTHON_VERSION%%", "-m", "compileall", "-q", "-o", "0", "-o", "1", "-o", "2", "%%PYTHON_SITELIBDIR%%"})
EOS
}

11
lang/python311/Makefile
View File

@ -1,5 +1,6 @@
PORTNAME= python
DISTVERSION= ${PYTHON_DISTVERSION}
PORTREVISION= 1
CATEGORIES= lang python
MASTER_SITES= PYTHON/ftp/python/${DISTVERSION:C/[a-z].*//}
PKGNAMESUFFIX= ${PYTHON_SUFFIX}
@ -14,8 +15,8 @@ LICENSE= PSFL
LIB_DEPENDS= libffi.so:devel/libffi
USES= compiler:c11 cpe ncurses pathfix pkgconfig readline \
shebangfix ssl tar:xz
USES= compiler:c11 cpe ncurses pathfix pkgconfig \
python:${PYTHON_DISTVERSION:R},env readline shebangfix ssl tar:xz trigger
PATHFIX_MAKEFILEIN= Makefile.pre.in
USE_LDCONFIG= yes
GNU_CONFIGURE= yes
@ -24,11 +25,7 @@ SHEBANG_FILES= Lib/*.py Lib/*/*.py Lib/*/*/*.py Lib/*/*/*/*.py
SHEBANG_FILES+= Lib/test/ziptestdata/exe_with_z64 \
Lib/test/ziptestdata/exe_with_zip \
Lib/test/ziptestdata/header.sh
# Duplicate python.mk variables. TODO: Let lang/python?? ports use python.mk bits.
PYTHON_VER= ${PYTHON_DISTVERSION:R}
PYTHON_VERSION= python${PYTHON_VER}
PYTHON_SUFFIX= ${PYTHON_VER:S/.//g}
TRIGGERS= ${PYTHON_VERSION}
DISABLED_EXTENSIONS= _sqlite3 _tkinter _gdbm
CONFIGURE_ARGS+= --enable-shared --without-ensurepip \

40
lang/python311/files/python3.11.ucl.in Normal file
View File

@ -0,0 +1,40 @@
path_glob: "%%PYTHON_SITELIBDIR%%/*"
trigger: {
type: lua
sandbox: false
script: <<EOS
function cleanup(directory)
for _,d in ipairs(pkg.readdir(directory)) do
local full_path = directory .. "/" .. d
local stat = pkg.stat(full_path)
if stat["type"] == "dir" then
if (d ~= "__pycache__") then
cleanup(full_path)
else
for _,bytecode_file in ipairs(pkg.readdir(full_path)) do
local file_origin = string.gsub(bytecode_file, "[.]cpython[-]%%PYTHON_SUFFIX%%[.].*pyc", ".py")
if file_origin then
local origin_path = directory .. "/" .. file_origin
if (not pkg.stat(origin_path)) then
--print(" >=> removed stale bytecode " .. bytecode_file)
os.remove(full_path .. "/" .. bytecode_file)
end
end
end
end
local res = pkg.readdir(full_path)
if #res == 0 then
--print(" >=> removed empty directory " .. full_path )
os.remove(full_path)
end
end
end
end
print(">=> Cleaning stale bytecode files...")
cleanup("%%PYTHON_SITELIBDIR%%")
print(">=> Byte-compiling Python source files...")
pkg.exec({"%%PYTHON_VERSION%%", "-m", "compileall", "-q", "-o", "0", "-o", "1", "-o", "2", "%%PYTHON_SITELIBDIR%%"})
EOS
}

10
lang/python37/Makefile
View File

@ -1,5 +1,6 @@
PORTNAME= python
DISTVERSION= ${PYTHON_DISTVERSION}
PORTREVISION= 1
CATEGORIES= lang python
MASTER_SITES= PYTHON/ftp/python/${DISTVERSION}
PKGNAMESUFFIX= ${PYTHON_SUFFIX}
@ -18,17 +19,14 @@ EXPIRATION_DATE= 2023-06-27
LIB_DEPENDS= libffi.so:devel/libffi \
libmpdec.so:math/mpdecimal
USES= cpe ncurses pathfix pkgconfig readline shebangfix ssl tar:xz
USES= cpe ncurses pathfix pkgconfig python:${PYTHON_DISTVERSION:R},env readline \
shebangfix ssl tar:xz trigger
PATHFIX_MAKEFILEIN= Makefile.pre.in
USE_LDCONFIG= yes
GNU_CONFIGURE= yes
python_CMD= ${PREFIX}/bin/python${PYTHON_DISTVERSION:R}
SHEBANG_FILES= Lib/*.py Lib/*/*.py Lib/*/*/*.py Lib/*/*/*/*.py
# Duplicate python.mk variables. TODO: Let lang/python?? ports use python.mk bits.
PYTHON_VER= ${PYTHON_DISTVERSION:R}
PYTHON_VERSION= python${PYTHON_VER}
PYTHON_SUFFIX= ${PYTHON_VER:S/.//g}
TRIGGERS= ${PYTHON_VERSION}
DISABLED_EXTENSIONS= _sqlite3 _tkinter _gdbm
CONFIGURE_ARGS+= --enable-shared --with-system-ffi --with-system-libmpdec --without-ensurepip

42
lang/python37/files/python3.7.ucl.in Normal file
View File

@ -0,0 +1,42 @@
path_glob: "%%PYTHON_SITELIBDIR%%/*"
trigger: {
type: lua
sandbox: false
script: <<EOS
function cleanup(directory)
for _,d in ipairs(pkg.readdir(directory)) do
local full_path = directory .. "/" .. d
local stat = pkg.stat(full_path)
if stat["type"] == "dir" then
if (d ~= "__pycache__") then
cleanup(full_path)
else
for _,bytecode_file in ipairs(pkg.readdir(full_path)) do
local file_origin = string.gsub(bytecode_file, "[.]cpython[-]%%PYTHON_SUFFIX%%[.].*pyc", ".py")
if file_origin then
local origin_path = directory .. "/" .. file_origin
if (not pkg.stat(origin_path)) then
--print(" >=> removed stale bytecode " .. bytecode_file)
os.remove(full_path .. "/" .. bytecode_file)
end
end
end
end
local res = pkg.readdir(full_path)
if #res == 0 then
--print(" >=> removed empty directory " .. full_path )
os.remove(full_path)
end
end
end
end
print(">=> Cleaning stale bytecode files...")
cleanup("%%PYTHON_SITELIBDIR%%")
print(">=> Byte-compiling Python source files...")
pkg.exec({"%%PYTHON_VERSION%%", "-m", "compileall", "-q", "%%PYTHON_SITELIBDIR%%"})
pkg.exec({"%%PYTHON_VERSION%%", "-O", "-m", "compileall", "-q", "%%PYTHON_SITELIBDIR%%"})
pkg.exec({"%%PYTHON_VERSION%%", "-OO", "-m", "compileall", "-q", "%%PYTHON_SITELIBDIR%%"})
EOS
}

10
lang/python38/Makefile
View File

@ -1,5 +1,6 @@
PORTNAME= python
DISTVERSION= ${PYTHON_DISTVERSION}
PORTREVISION= 1
CATEGORIES= lang python
MASTER_SITES= PYTHON/ftp/python/${DISTVERSION}
PKGNAMESUFFIX= ${PYTHON_SUFFIX}
@ -14,7 +15,8 @@ LICENSE= PSFL
LIB_DEPENDS= libffi.so:devel/libffi
USES= cpe ncurses pathfix pkgconfig readline shebangfix ssl tar:xz
USES= cpe ncurses pathfix pkgconfig python:${PYTHON_DISTVERSION:R},env readline \
shebangfix ssl tar:xz trigger
PATHFIX_MAKEFILEIN= Makefile.pre.in
USE_LDCONFIG= yes
GNU_CONFIGURE= yes
@ -23,11 +25,7 @@ SHEBANG_FILES= Lib/*.py Lib/*/*.py Lib/*/*/*.py Lib/*/*/*/*.py
SHEBANG_FILES+= Lib/test/ziptestdata/exe_with_z64 \
Lib/test/ziptestdata/exe_with_zip \
Lib/test/ziptestdata/header.sh
# Duplicate python.mk variables. TODO: Let lang/python?? ports use python.mk bits.
PYTHON_VER= ${PYTHON_DISTVERSION:R}
PYTHON_VERSION= python${PYTHON_VER}
PYTHON_SUFFIX= ${PYTHON_VER:S/.//g}
TRIGGERS= ${PYTHON_VERSION}
DISABLED_EXTENSIONS= _sqlite3 _tkinter _gdbm
CONFIGURE_ARGS+= --enable-shared --without-ensurepip \

42
lang/python38/files/python3.8.ucl.in Normal file
View File

@ -0,0 +1,42 @@
path_glob: "%%PYTHON_SITELIBDIR%%/*"
trigger: {
type: lua
sandbox: false
script: <<EOS
function cleanup(directory)
for _,d in ipairs(pkg.readdir(directory)) do
local full_path = directory .. "/" .. d
local stat = pkg.stat(full_path)
if stat["type"] == "dir" then
if (d ~= "__pycache__") then
cleanup(full_path)
else
for _,bytecode_file in ipairs(pkg.readdir(full_path)) do
local file_origin = string.gsub(bytecode_file, "[.]cpython[-]%%PYTHON_SUFFIX%%[.].*pyc", ".py")
if file_origin then
local origin_path = directory .. "/" .. file_origin
if (not pkg.stat(origin_path)) then
--print(" >=> removed stale bytecode " .. bytecode_file)
os.remove(full_path .. "/" .. bytecode_file)
end
end
end
end
local res = pkg.readdir(full_path)
if #res == 0 then
--print(" >=> removed empty directory " .. full_path )
os.remove(full_path)
end
end
end
end
print(">=> Cleaning stale bytecode files...")
cleanup("%%PYTHON_SITELIBDIR%%")
print(">=> Byte-compiling Python source files...")
pkg.exec({"%%PYTHON_VERSION%%", "-m", "compileall", "-q", "%%PYTHON_SITELIBDIR%%"})
pkg.exec({"%%PYTHON_VERSION%%", "-O", "-m", "compileall", "-q", "%%PYTHON_SITELIBDIR%%"})
pkg.exec({"%%PYTHON_VERSION%%", "-OO", "-m", "compileall", "-q", "%%PYTHON_SITELIBDIR%%"})
EOS
}

11
lang/python39/Makefile
View File

@ -1,5 +1,6 @@
PORTNAME= python
DISTVERSION= ${PYTHON_DISTVERSION}
PORTREVISION= 1
CATEGORIES= lang python
MASTER_SITES= PYTHON/ftp/python/${DISTVERSION}
PKGNAMESUFFIX= ${PYTHON_SUFFIX}
@ -14,8 +15,8 @@ LICENSE= PSFL
LIB_DEPENDS= libffi.so:devel/libffi
USES= compiler:c11 cpe ncurses pathfix pkgconfig readline \
shebangfix ssl tar:xz
USES= compiler:c11 cpe ncurses pathfix pkgconfig \
python:${PYTHON_DISTVERSION:R},env readline shebangfix ssl tar:xz trigger
PATHFIX_MAKEFILEIN= Makefile.pre.in
USE_LDCONFIG= yes
GNU_CONFIGURE= yes
@ -24,11 +25,7 @@ SHEBANG_FILES= Lib/*.py Lib/*/*.py Lib/*/*/*.py Lib/*/*/*/*.py
SHEBANG_FILES+= Lib/test/ziptestdata/exe_with_z64 \
Lib/test/ziptestdata/exe_with_zip \
Lib/test/ziptestdata/header.sh
# Duplicate python.mk variables. TODO: Let lang/python?? ports use python.mk bits.
PYTHON_VER= ${PYTHON_DISTVERSION:R}
PYTHON_VERSION= python${PYTHON_VER}
PYTHON_SUFFIX= ${PYTHON_VER:S/.//g}
TRIGGERS= ${PYTHON_VERSION}
DISABLED_EXTENSIONS= _sqlite3 _tkinter _gdbm
CONFIGURE_ARGS+= --enable-shared --without-ensurepip \

40
lang/python39/files/python3.9.ucl.in Normal file
View File

@ -0,0 +1,40 @@
path_glob: "%%PYTHON_SITELIBDIR%%/*"
trigger: {
type: lua
sandbox: false
script: <<EOS
function cleanup(directory)
for _,d in ipairs(pkg.readdir(directory)) do
local full_path = directory .. "/" .. d
local stat = pkg.stat(full_path)
if stat["type"] == "dir" then
if (d ~= "__pycache__") then
cleanup(full_path)
else
for _,bytecode_file in ipairs(pkg.readdir(full_path)) do
local file_origin = string.gsub(bytecode_file, "[.]cpython[-]%%PYTHON_SUFFIX%%[.].*pyc", ".py")
if file_origin then
local origin_path = directory .. "/" .. file_origin
if (not pkg.stat(origin_path)) then
--print(" >=> removed stale bytecode " .. bytecode_file)
os.remove(full_path .. "/" .. bytecode_file)
end
end
end
end
local res = pkg.readdir(full_path)
if #res == 0 then
--print(" >=> removed empty directory " .. full_path )
os.remove(full_path)
end
end
end
end
print(">=> Cleaning stale bytecode files...")
cleanup("%%PYTHON_SITELIBDIR%%")
print(">=> Byte-compiling Python source files...")
pkg.exec({"%%PYTHON_VERSION%%", "-m", "compileall", "-q", "-o", "0", "-o", "1", "-o", "2", "%%PYTHON_SITELIBDIR%%"})
EOS
}