From 34711ba5cfbe6eb28bef5f53aca02d622880ba59 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Tue, 6 Jun 2023 14:51:09 +0200 Subject: [PATCH 1/4] kaldi: Fix build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It looks like after a bump, CMake started passing `--git-dir=.git` as the first argument of `git rev-parse`. But we do not really need to spoof `git` program now that Kaldi uses CMake’s `FetchContent` module. We can just pass the path using configure flag directly: https://cmake.org/cmake/help/latest/module/FetchContent.html#variable:FETCHCONTENT_SOURCE_DIR_%3CuppercaseName%3E Git was also inadvertently used for generating `KALDI_PATCH_NUMBER` in `VersionHelper`, to be potentially appended to `KALDI_VERSION` but it was not actually being done by default. We can set the variable directly to skip `VersionHelper`. Also remove redundant `enableParallelBuild` since CMake enables it. And drop separate `dev` output since Kaldi currently hardcodes the `include/` directory so consumers would not be able to find them. Splitting it out only removes 7 out of 303 MB. --- pkgs/tools/audio/kaldi/default.nix | 42 ++++++++---------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/pkgs/tools/audio/kaldi/default.nix b/pkgs/tools/audio/kaldi/default.nix index 79da94f72435..6c67924f9eea 100644 --- a/pkgs/tools/audio/kaldi/default.nix +++ b/pkgs/tools/audio/kaldi/default.nix @@ -39,39 +39,9 @@ stdenv.mkDerivation { "-DBUILD_SHARED_LIBS=on" "-DBLAS_LIBRARIES=-lblas" "-DLAPACK_LIBRARIES=-llapack" + "-DFETCHCONTENT_SOURCE_DIR_OPENFST:PATH=${openfst}" ]; - enableParallelBuilding = true; - - preConfigure = '' - mkdir bin - cat > bin/git <<'EOF' - #!${stdenv.shell} - if [[ "$1" == "--version" ]]; then - # cmake checks this - ${git}/bin/git --version - elif [[ "$1" == "clone" ]]; then - # mock this call: - - # https://github.com/kaldi-asr/kaldi/blob/c9d8b9ad3fef89237ba5517617d977b7d70a7ed5/cmake/third_party/openfst.cmake#L5 - cp -r ${openfst} ''${@: -1} - chmod -R +w ''${@: -1} - elif [[ "$1" == "rev-list" ]]; then - # fix up this call: - # https://github.com/kaldi-asr/kaldi/blob/c9d8b9ad3fef89237ba5517617d977b7d70a7ed5/cmake/VersionHelper.cmake#L8 - echo 0 - elif [[ "$1" == "rev-parse" ]]; then - echo ${openfst.rev} - echo 0 - fi - true - EOF - chmod +x bin/git - export PATH=$(pwd)/bin:$PATH - ''; - - outputs = [ "out" "dev" ]; - buildInputs = [ openblas openfst @@ -86,6 +56,16 @@ stdenv.mkDerivation { python3 ]; + preConfigure = '' + cmakeFlagsArray+=( + # Extract version without the need for git. + # https://github.com/kaldi-asr/kaldi/blob/71f38e62cad01c3078555bfe78d0f3a527422d75/cmake/VersionHelper.cmake + # Patch number is not actually used by default so we can just ignore it. + # https://github.com/kaldi-asr/kaldi/blob/71f38e62cad01c3078555bfe78d0f3a527422d75/CMakeLists.txt#L214 + "-DKALDI_VERSION=$(cat src/.version)" + ) + ''; + postInstall = '' mkdir -p $out/share/kaldi cp -r ../egs $out/share/kaldi From e8588071cc2905a13c164292bd7e93ca29d1031c Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Tue, 6 Jun 2023 20:42:58 +0200 Subject: [PATCH 2/4] kaldi: Remove openfst from inputs It is shadowed by fetchgit source and can only be used vendored. --- pkgs/tools/audio/kaldi/default.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkgs/tools/audio/kaldi/default.nix b/pkgs/tools/audio/kaldi/default.nix index 6c67924f9eea..2196c4bcccb0 100644 --- a/pkgs/tools/audio/kaldi/default.nix +++ b/pkgs/tools/audio/kaldi/default.nix @@ -3,7 +3,6 @@ , openblas , blas , lapack -, openfst , icu , cmake , pkg-config @@ -44,7 +43,6 @@ stdenv.mkDerivation { buildInputs = [ openblas - openfst icu ] ++ lib.optionals stdenv.isDarwin [ Accelerate From 5abf11a956794ba5245fb7636bf9b895d5a5281b Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Tue, 6 Jun 2023 20:54:40 +0200 Subject: [PATCH 3/4] kaldi: Add update script Also switch to finalAttrs so that the values can be easily overridden. --- pkgs/tools/audio/kaldi/default.nix | 45 ++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/pkgs/tools/audio/kaldi/default.nix b/pkgs/tools/audio/kaldi/default.nix index 2196c4bcccb0..9e4b03088e21 100644 --- a/pkgs/tools/audio/kaldi/default.nix +++ b/pkgs/tools/audio/kaldi/default.nix @@ -10,19 +10,15 @@ , git , python3 , Accelerate +, _experimental-update-script-combinators +, common-updater-scripts +, ripgrep +, unstableGitUpdater +, writeShellScript }: assert blas.implementation == "openblas" && lapack.implementation == "openblas"; -let - # rev from https://github.com/kaldi-asr/kaldi/blob/master/cmake/third_party/openfst.cmake - openfst = fetchFromGitHub { - owner = "kkm000"; - repo = "openfst"; - rev = "338225416178ac36b8002d70387f5556e44c8d05"; - sha256 = "sha256-MGEUuw7ex+WcujVdxpO2Bf5sB6Z0edcAeLGqW/Lo1Hs="; - }; -in -stdenv.mkDerivation { +stdenv.mkDerivation (finalAttrs: { pname = "kaldi"; version = "unstable-2022-09-26"; @@ -38,7 +34,7 @@ stdenv.mkDerivation { "-DBUILD_SHARED_LIBS=on" "-DBLAS_LIBRARIES=-lblas" "-DLAPACK_LIBRARIES=-llapack" - "-DFETCHCONTENT_SOURCE_DIR_OPENFST:PATH=${openfst}" + "-DFETCHCONTENT_SOURCE_DIR_OPENFST:PATH=${finalAttrs.passthru.sources.openfst}" ]; buildInputs = [ @@ -69,6 +65,31 @@ stdenv.mkDerivation { cp -r ../egs $out/share/kaldi ''; + passthru = { + sources = { + # rev from https://github.com/kaldi-asr/kaldi/blob/master/cmake/third_party/openfst.cmake + openfst = fetchFromGitHub { + owner = "kkm000"; + repo = "openfst"; + rev = "338225416178ac36b8002d70387f5556e44c8d05"; + hash = "sha256-MGEUuw7ex+WcujVdxpO2Bf5sB6Z0edcAeLGqW/Lo1Hs="; + }; + }; + + updateScript = + let + updateSource = unstableGitUpdater {}; + updateOpenfst = writeShellScript "update-openfst" '' + hash=$(${ripgrep}/bin/rg --multiline --pcre2 --only-matching 'FetchContent_Declare\(\s*openfst[^)]*GIT_TAG\s*([0-9a-f]{40})' --replace '$1' "${finalAttrs.src}/cmake/third_party/openfst.cmake") + ${common-updater-scripts}/bin/update-source-version kaldi.sources.openfst "$hash" --source-key=out "--version-key=rev" + ''; + in + _experimental-update-script-combinators.sequence [ + updateSource + updateOpenfst + ]; + }; + meta = with lib; { description = "Speech Recognition Toolkit"; homepage = "https://kaldi-asr.org"; @@ -76,4 +97,4 @@ stdenv.mkDerivation { maintainers = with maintainers; [ mic92 ]; platforms = platforms.unix; }; -} +}) From 2024f7d11d5c9df21728ba5c15ffac370692bb29 Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Tue, 6 Jun 2023 20:57:14 +0200 Subject: [PATCH 4/4] =?UTF-8?q?kaldi:=20unstable-2022-09-26=20=E2=86=92=20?= =?UTF-8?q?unstable-2023-05-02?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/kaldi-asr/kaldi/compare/f6f4ccaf213f0fe8b26e633a7dc0c802150626a0..71f38e62cad01c3078555bfe78d0f3a527422d75 --- pkgs/tools/audio/kaldi/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/tools/audio/kaldi/default.nix b/pkgs/tools/audio/kaldi/default.nix index 9e4b03088e21..777a7535a10c 100644 --- a/pkgs/tools/audio/kaldi/default.nix +++ b/pkgs/tools/audio/kaldi/default.nix @@ -20,13 +20,13 @@ assert blas.implementation == "openblas" && lapack.implementation == "openblas"; stdenv.mkDerivation (finalAttrs: { pname = "kaldi"; - version = "unstable-2022-09-26"; + version = "unstable-2023-05-02"; src = fetchFromGitHub { owner = "kaldi-asr"; repo = "kaldi"; - rev = "f6f4ccaf213f0fe8b26e633a7dc0c802150626a0"; - sha256 = "sha256-ybW2J4lWf6YaQGZZvxEVDUMAg84DC17W+yX6ZsuBDac="; + rev = "71f38e62cad01c3078555bfe78d0f3a527422d75"; + sha256 = "sha256-2xm0F80cjovy/G9Ytq/iwa1eexZk0mromv6PPuNIT8U="; }; cmakeFlags = [