python3Packages.dedupe: init at 3.0.3 (#453792)

pkgs/development/python-modules/affinegap/default.nix

@@ -0,0 +1,50 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+
+  # build-system
+  cython,
+  setuptools,
+
+  # tests
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "affinegap";
+  version = "1.12";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "dedupeio";
+    repo = "affinegap";
+    tag = "v${version}";
+    hash = "sha256-9eX41eoME5Vdtq+c04eQbMYnViy6QKOhKkafrkeMylI=";
+  };
+
+  build-system = [
+    cython
+    setuptools
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  # Prevent importing from source during test collection (only $out has compiled extensions)
+  preCheck = ''
+    rm -rf affinegap
+  '';
+
+  pythonImportsCheck = [
+    "affinegap"
+  ];
+
+  meta = {
+    description = "Cython implementation of the affine gap string distance";
+    homepage = "https://github.com/dedupeio/affinegap";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ daniel-fahey ];
+  };
+}

pkgs/development/python-modules/categorical-distance/default.nix

@@ -0,0 +1,53 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  unstableGitUpdater,
+
+  # build-system
+  setuptools,
+
+  # dependencies
+  numpy,
+
+  # tests
+  pytestCheckHook,
+}:
+
+buildPythonPackage {
+  pname = "categorical-distance";
+  version = "1.9-unstable-2020-03-31";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "dedupeio";
+    repo = "categorical-distance";
+    rev = "07d079fd412ccf06cdb200b3cd2cfa4b67f78722";
+    hash = "sha256-zSjSrlFiRus/T2XZdakLQpF1u/LV0VNWwrc8lhss6kU=";
+  };
+
+  build-system = [
+    setuptools
+  ];
+
+  dependencies = [
+    numpy
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "categorical"
+  ];
+
+  passthru.updateScript = unstableGitUpdater { };
+
+  meta = {
+    description = "Compare similarity of categorical variables using Jaccard index";
+    homepage = "https://github.com/dedupeio/categorical-distance";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ daniel-fahey ];
+  };
+}

pkgs/development/python-modules/dedupe-levenshtein-search/default.nix

@@ -0,0 +1,45 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+
+  # build-system
+  setuptools,
+
+  # tests
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "dedupe-levenshtein-search";
+  version = "1.4.5";
+  pyproject = true;
+
+  # NOTE: This is a fork of mattandahalfew/Levenshtein_search created for MIT licensing.
+  # TODO: Evaluate if upstream version could be used instead.
+  src = fetchFromGitHub {
+    owner = "dedupeio";
+    repo = "Levenshtein_search";
+    tag = "v${version}";
+    hash = "sha256-YhsZA28H4OUkQEBtJ+9OXJld4Z/PJbOPqAQQ9qaXSjk=";
+  };
+
+  build-system = [
+    setuptools
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "Levenshtein_search"
+  ];
+
+  meta = {
+    description = "Search through documents for approximately matching strings using Levenshtein distance";
+    homepage = "https://github.com/dedupeio/Levenshtein_search";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ daniel-fahey ];
+  };
+}

pkgs/development/python-modules/dedupe-pylbfgs/default.nix

@@ -0,0 +1,59 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+
+  # build-system
+  cython,
+  numpy,
+  setuptools,
+
+  # tests
+  pytest-cov-stub,
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "dedupe-pylbfgs";
+  version = "0.2.0.16";
+  pyproject = true;
+
+  # NOTE: This is a fork of larsmans/pylbfgs maintained by dedupeio
+  src = fetchFromGitHub {
+    owner = "dedupeio";
+    repo = "pylbfgs";
+    tag = "${version}";
+    hash = "sha256-H416dgZQxyqsnhmlK5keW8cJWY6gea4mebVuP0IEVOU=";
+  };
+
+  build-system = [
+    cython
+    numpy
+    setuptools
+  ];
+
+  dependencies = [
+    numpy
+  ];
+
+  nativeCheckInputs = [
+    pytest-cov-stub
+    pytestCheckHook
+  ];
+
+  # Prevent importing from source during test collection (only $out has compiled extensions)
+  preCheck = ''
+    rm -rf lbfgs
+  '';
+
+  pythonImportsCheck = [
+    "lbfgs"
+  ];
+
+  meta = {
+    description = "Python wrapper for L-BFGS and OWL-QN optimization algorithms";
+    homepage = "https://github.com/dedupeio/pylbfgs";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ daniel-fahey ];
+  };
+}

pkgs/development/python-modules/dedupe/default.nix

@@ -0,0 +1,106 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+
+  # build-system
+  cython,
+  setuptools,
+
+  # dependencies
+  affinegap,
+  btrees,
+  categorical-distance,
+  dedupe-levenshtein-search,
+  doublemetaphone,
+  haversine,
+  highered,
+  numpy,
+  scikit-learn,
+  simplecosine,
+  zope-index,
+  dedupe,
+
+  # tests
+  pytest-cov-stub,
+  pytestCheckHook,
+  python,
+  runCommand,
+}:
+
+buildPythonPackage rec {
+  pname = "dedupe";
+  version = "3.0.3";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "dedupeio";
+    repo = "dedupe";
+    tag = "v${version}";
+    hash = "sha256-tfBJeaeZw5w5OwM+AOfy9H6P2zbShjN/kuzEbpxATHI=";
+  };
+
+  build-system = [
+    cython
+    setuptools
+  ];
+
+  dependencies = [
+    affinegap
+    btrees
+    categorical-distance
+    dedupe-levenshtein-search
+    doublemetaphone
+    haversine
+    highered
+    numpy
+    scikit-learn
+    simplecosine
+    zope-index
+  ];
+
+  nativeCheckInputs = [
+    pytest-cov-stub
+    pytestCheckHook
+  ];
+
+  # Remove source directory so pytest imports compiled extension from $out
+  preCheck = ''
+    rm -rf dedupe
+  '';
+
+  pythonImportsCheck = [
+    "dedupe"
+  ];
+
+  passthru.tests.benchmarks =
+    runCommand "dedupe-benchmarks-test"
+      {
+        nativeBuildInputs = [ (python.withPackages (ps: [ dedupe ])) ];
+      }
+      ''
+        # Copy benchmarks to writable location
+        cp -r ${src}/benchmarks benchmarks
+        chmod -R +w benchmarks
+        cd benchmarks
+
+        # Run all three canonical benchmarks
+        for benchmark in canonical canonical_gazetteer canonical_matching; do
+          echo "Running $benchmark benchmark..."
+          # Redirect stderr to /dev/null (`2>/dev/null`) to suppress Python 3.13
+          # multiprocessing resource tracker warnings from scikit-learn/joblib subprocesses
+          # `|| exit 1` provides fail-fast behavior: exit immediately if any benchmark fails
+          PYTHONPATH=$PWD python -m benchmarks.$benchmark 2>/dev/null || exit 1
+        done
+
+        touch $out
+      '';
+
+  meta = {
+    description = "Library for accurate and scalable fuzzy matching, deduplication and entity resolution";
+    homepage = "https://github.com/dedupeio/dedupe";
+    changelog = "https://github.com/dedupeio/dedupe/blob/${src.tag}/CHANGELOG.md";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ daniel-fahey ];
+  };
+}

pkgs/development/python-modules/doublemetaphone/default.nix

@@ -0,0 +1,50 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+
+  # build-system
+  cython,
+  setuptools,
+
+  # tests
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "doublemetaphone";
+  version = "1.2i";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "dedupeio";
+    repo = "doublemetaphone";
+    tag = "v${version}";
+    hash = "sha256-VPJqHxQHLiLSko+aJYTIgISluHPARgQN5pYWYxP9QKQ=";
+  };
+
+  build-system = [
+    cython
+    setuptools
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  # Prevent importing from source during test collection (only $out has compiled extensions)
+  preCheck = ''
+    rm -rf doublemetaphone
+  '';
+
+  pythonImportsCheck = [
+    "doublemetaphone"
+  ];
+
+  meta = {
+    description = "Python wrapper for Double Metaphone phonetic encoding algorithm";
+    homepage = "https://github.com/dedupeio/doublemetaphone";
+    license = lib.licenses.artistic1;
+    maintainers = with lib.maintainers; [ daniel-fahey ];
+  };
+}

pkgs/development/python-modules/highered/default.nix

@@ -0,0 +1,58 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  unstableGitUpdater,
+
+  # build-system
+  setuptools,
+
+  # dependencies
+  pyhacrf-datamade,
+  numpy,
+
+  # tests
+  pytestCheckHook,
+}:
+
+buildPythonPackage {
+  pname = "highered";
+  version = "0.2.1-unstable-2020-03-31";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "dedupeio";
+    repo = "highered";
+    rev = "5e6e505e182ff91b1620535a491ad4a3d98ef71e";
+    hash = "sha256-sDOAB0QabJ/WJYSIZ31J12kSDQADQUilE2SmGPjXmZo=";
+  };
+
+  build-system = [
+    setuptools
+  ];
+
+  dependencies = [
+    pyhacrf-datamade
+    numpy
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  # No tests in repository
+  doCheck = false;
+
+  pythonImportsCheck = [
+    "highered"
+  ];
+
+  passthru.updateScript = unstableGitUpdater { };
+
+  meta = {
+    description = "Learnable edit distance using CRF (Conditional Random Fields)";
+    homepage = "https://github.com/dedupeio/highered";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ daniel-fahey ];
+  };
+}

pkgs/development/python-modules/pyhacrf-datamade/default.nix

@@ -0,0 +1,71 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  unstableGitUpdater,
+
+  # build-system
+  cython,
+  numpy,
+  setuptools,
+
+  # dependencies
+  dedupe-pylbfgs,
+
+  # tests
+  pytestCheckHook,
+}:
+
+buildPythonPackage {
+  pname = "pyhacrf-datamade";
+  # Tagged release requested upstream in https://github.com/dedupeio/pyhacrf/issues/57
+  version = "0.2.8-unstable-2025-05-16";
+  pyproject = true;
+
+  # NOTE: This is a fork of dirko/pyhacrf maintained by dedupeio
+  src = fetchFromGitHub {
+    owner = "dedupeio";
+    repo = "pyhacrf";
+    rev = "899aa6c2c48e5afe8fb40727ffd6070e4ba71c31";
+    hash = "sha256-MVkOChDblu7A/ve51SYqO7lNoTXwh37bHjnZd+NvzK0=";
+  };
+
+  build-system = [
+    cython
+    numpy
+    setuptools
+  ];
+
+  dependencies = [
+    dedupe-pylbfgs
+    numpy
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  # Change to temp directory so pytest imports from installed package in $out, not source
+  preCheck = ''
+    export TEST_DIR=$(mktemp -d)
+    cp -r pyhacrf/tests $TEST_DIR/
+    pushd $TEST_DIR
+  '';
+
+  postCheck = ''
+    popd
+  '';
+
+  pythonImportsCheck = [
+    "pyhacrf"
+  ];
+
+  passthru.updateScript = unstableGitUpdater { };
+
+  meta = {
+    description = "Hidden alignment conditional random field for classifying string pairs";
+    homepage = "https://github.com/dedupeio/pyhacrf";
+    license = lib.licenses.bsd3;
+    maintainers = with lib.maintainers; [ daniel-fahey ];
+  };
+}

pkgs/development/python-modules/simplecosine/default.nix

@@ -0,0 +1,50 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+
+  # build-system
+  setuptools,
+
+  # dependencies
+  numpy,
+
+  # tests
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "simplecosine";
+  version = "1.2";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "dedupeio";
+    repo = "simplecosine";
+    tag = "v${version}";
+    hash = "sha256-TNQnSbCh7o5JsxvfljRGSNwptwpLHmVw9gyk0TELDek=";
+  };
+
+  build-system = [
+    setuptools
+  ];
+
+  dependencies = [
+    numpy
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "simplecosine"
+  ];
+
+  meta = {
+    description = "Simple cosine distance calculation for string comparison";
+    homepage = "https://github.com/dedupeio/simplecosine";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ daniel-fahey ];
+  };
+}

pkgs/development/python-modules/zope-index/default.nix

@@ -0,0 +1,64 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+
+  # build-system
+  setuptools,
+
+  # dependencies
+  btrees,
+  persistent,
+  zope-interface,
+
+  # tests
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "zope-index";
+  version = "8.0";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "zopefoundation";
+    repo = "zope.index";
+    tag = "${version}";
+    hash = "sha256-dHapd/+pJh6qzVx9FGSMmPsGbz8NhAoPqufXm3FOuM8=";
+  };
+
+  postPatch = ''
+    substituteInPlace pyproject.toml \
+      --replace-fail "setuptools ==" "setuptools >="
+  '';
+
+  build-system = [
+    setuptools
+  ];
+
+  dependencies = [
+    btrees
+    persistent
+    zope-interface
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "zope.index"
+  ];
+
+  pythonNamespaces = [
+    "zope"
+  ];
+
+  meta = {
+    description = "Full-text indexing and searching for Zope";
+    homepage = "https://github.com/zopefoundation/zope.index";
+    changelog = "https://github.com/zopefoundation/zope.index/blob/${version}/CHANGES.rst";
+    license = lib.licenses.zpl21;
+    maintainers = with lib.maintainers; [ daniel-fahey ];
+  };
+}

pkgs/top-level/python-packages.nix

@@ -124,6 +124,8 @@ self: super: with self; {
 
   affine-gaps = callPackage ../development/python-modules/affine-gaps { };
 
+  affinegap = callPackage ../development/python-modules/affinegap { };
+
   afsapi = callPackage ../development/python-modules/afsapi { };
 
   agate = callPackage ../development/python-modules/agate { };
@@ -2409,6 +2411,8 @@ self: super: with self; {
     };
   };
 
+  categorical-distance = callPackage ../development/python-modules/categorical-distance { };
+
   catkin-pkg = callPackage ../development/python-modules/catkin-pkg { };
 
   catppuccin = callPackage ../development/python-modules/catppuccin { };
@@ -3552,6 +3556,12 @@ self: super: with self; {
 
   decorator = callPackage ../development/python-modules/decorator { };
 
+  dedupe = callPackage ../development/python-modules/dedupe { };
+
+  dedupe-levenshtein-search = callPackage ../development/python-modules/dedupe-levenshtein-search { };
+
+  dedupe-pylbfgs = callPackage ../development/python-modules/dedupe-pylbfgs { };
+
   deebot-client = callPackage ../development/python-modules/deebot-client { };
 
   deemix = callPackage ../development/python-modules/deemix { };
@@ -4391,6 +4401,8 @@ self: super: with self; {
 
   dotwiz = callPackage ../development/python-modules/dotwiz { };
 
+  doublemetaphone = callPackage ../development/python-modules/doublemetaphone { };
+
   doubleratchet = callPackage ../development/python-modules/doubleratchet { };
 
   doubles = callPackage ../development/python-modules/doubles { };
@@ -6670,6 +6682,8 @@ self: super: with self; {
 
   highdicom = callPackage ../development/python-modules/highdicom { };
 
+  highered = callPackage ../development/python-modules/highered { };
+
   highspy = callPackage ../development/python-modules/highspy { };
 
   hightime = callPackage ../development/python-modules/hightime { };
@@ -13203,6 +13217,8 @@ self: super: with self; {
 
   pygtrie = callPackage ../development/python-modules/pygtrie { };
 
+  pyhacrf-datamade = callPackage ../development/python-modules/pyhacrf-datamade { };
+
   pyhamcrest = callPackage ../development/python-modules/pyhamcrest { };
 
   pyhanko = callPackage ../development/python-modules/pyhanko { };
@@ -16860,6 +16876,8 @@ self: super: with self; {
 
   simplebayes = callPackage ../development/python-modules/simplebayes { };
 
+  simplecosine = callPackage ../development/python-modules/simplecosine { };
+
   simpleeval = callPackage ../development/python-modules/simpleeval { };
 
   simplefin4py = callPackage ../development/python-modules/simplefin4py { };
@@ -20670,6 +20688,8 @@ self: super: with self; {
 
   zope-i18nmessageid = callPackage ../development/python-modules/zope-i18nmessageid { };
 
+  zope-index = callPackage ../development/python-modules/zope-index { };
+
   zope-interface = callPackage ../development/python-modules/zope-interface { };
 
   zope-lifecycleevent = callPackage ../development/python-modules/zope-lifecycleevent { };