From 23e348bfe2b161e761dd37e3be86a807f3cd6f94 Mon Sep 17 00:00:00 2001
From: Frederik Rietdijk <fridh@fridh.nl>
Date: Tue, 27 Apr 2021 11:42:10 +0200
Subject: [PATCH] python2 and python3: build unoptimized bytecode again

In 9d03ff52229f36ff4b4e9d0ee2db4a46b90512bb I made the CPython builds
reproducible. This required not generating default unoptimized bytecode.
I was under the impression the optimized bytecode would be used then,
but you need to opt-in on that. Not having the default bytecode resulted
in a significant performance hit. Therefore, bytecode is generated again
in this commit, and thereby the builds are no longer reproducible.

https://bugs.python.org/issue29708
---
 .../interpreters/python/cpython/2.7/default.nix  | 12 ++++++++----
 .../interpreters/python/cpython/default.nix      | 16 +++++++++++-----
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix
index 5ee88ebb8810..ad2b7fe68605 100644
--- a/pkgs/development/interpreters/python/cpython/2.7/default.nix
+++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix
@@ -27,9 +27,9 @@
 , sha256
 , passthruFun
 , static ? false
-, stripBytecode ? reproducibleBuild
+, stripBytecode ? true
 , rebuildBytecode ? true
-, reproducibleBuild ? true
+, reproducibleBuild ? false
 , enableOptimizations ? false
 , pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
 }:
@@ -48,6 +48,8 @@ assert lib.assertMsg (reproducibleBuild -> stripBytecode)
 assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
   "Deterministic builds are not achieved when optimizations are enabled.";
 
+assert lib.assertMsg (reproducibleBuild -> (!rebuildBytecode))
+  "Deterministic builds are not achieved when (default unoptimized) bytecode is created.";
 
 with lib;
 
@@ -296,8 +298,10 @@ in with passthru; stdenv.mkDerivation ({
         # First we delete all old bytecode.
         find $out -name "*.pyc" -delete
         '' + optionalString rebuildBytecode ''
-        # Then, we build for the two optimization levels.
-        # We do not build unoptimized bytecode, because its not entirely deterministic yet.
+        # We build 3 levels of optimized bytecode. Note the default level, without optimizations,
+        # is not reproducible yet. https://bugs.python.org/issue29708
+        # Not creating bytecode will result in a large performance loss however, so we do build it.
+        find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
         find $out -name "*.py" | ${pythonForBuildInterpreter} -O  -m compileall -q -f -x "lib2to3" -i -
         find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
       '' + optionalString stdenv.hostPlatform.isCygwin ''
diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix
index f52255f4748b..d59d3cb44708 100644
--- a/pkgs/development/interpreters/python/cpython/default.nix
+++ b/pkgs/development/interpreters/python/cpython/default.nix
@@ -35,7 +35,7 @@
 , stripTests ? false
 , stripTkinter ? false
 , rebuildBytecode ? true
-, stripBytecode ? reproducibleBuild
+, stripBytecode ? true
 , includeSiteCustomize ? true
 , static ? stdenv.hostPlatform.isStatic
 , enableOptimizations ? false
@@ -46,7 +46,7 @@
 # enabling LTO on 32bit arch causes downstream packages to fail when linking
 # enabling LTO on *-darwin causes python3 to fail when linking.
 , enableLTO ? stdenv.is64bit && stdenv.isLinux
-, reproducibleBuild ? true
+, reproducibleBuild ? false
 , pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
 }:
 
@@ -73,6 +73,9 @@ assert lib.assertMsg (reproducibleBuild -> stripBytecode)
 assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
   "Deterministic builds are not achieved when optimizations are enabled.";
 
+assert lib.assertMsg (reproducibleBuild -> (!rebuildBytecode))
+  "Deterministic builds are not achieved when (default unoptimized) bytecode is created.";
+
 with lib;
 
 let
@@ -422,11 +425,14 @@ in with passthru; stdenv.mkDerivation {
     # First we delete all old bytecode.
     find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
     '' + optionalString rebuildBytecode ''
-    # Then, we build for the two optimization levels.
-    # We do not build unoptimized bytecode, because its not entirely deterministic yet.
     # Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
-    # compileall uses this checked-hash method by default when `SOURCE_DATE_EPOCH` is set.
+    # compileall uses the therein introduced checked-hash method by default when
+    # `SOURCE_DATE_EPOCH` is set.
     # We exclude lib2to3 because that's Python 2 code which fails
+    # We build 3 levels of optimized bytecode. Note the default level, without optimizations,
+    # is not reproducible yet. https://bugs.python.org/issue29708
+    # Not creating bytecode will result in a large performance loss however, so we do build it.
+    find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
     find $out -name "*.py" | ${pythonForBuildInterpreter} -O  -m compileall -q -f -x "lib2to3" -i -
     find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
   '';