python312Packages.dvc: refactor (#386940)

pkgs/development/python-modules/dvc-oss/default.nix

@@ -0,0 +1,43 @@
+{
+  lib,
+  buildPythonPackage,
+  dvc-objects,
+  fetchPypi,
+  ossfs,
+  pythonOlder,
+  setuptools-scm,
+}:
+
+buildPythonPackage rec {
+  pname = "dvc-oss";
+  version = "3.0.0";
+  pyproject = true;
+
+  disabled = pythonOlder "3.9";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-EEf3NAIvzSuW0ysGv24JIc0KZYEPf8HpsPrCmhR7apo=";
+  };
+
+  # Prevent circular dependency
+  pythonRemoveDeps = [ "dvc" ];
+
+  build-system = [ setuptools-scm ];
+
+  dependencies = [
+    dvc-objects
+    ossfs
+  ];
+
+  # Circular dependency
+  # pythonImportsCheck = [ "dvc_ssh" ];
+
+  meta = with lib; {
+    description = "Alibaba OSS plugin for dvc";
+    homepage = "https://pypi.org/project/dvc-oss/";
+    changelog = "https://github.com/iterative/dvc-oss/releases/tag/${version}";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ fab ];
+  };
+}

pkgs/development/python-modules/dvc-ssh/default.nix

@@ -6,16 +6,20 @@
   fetchPypi,
   setuptools-scm,
   sshfs,
+  pythonOlder,
 }:
 
 buildPythonPackage rec {
   pname = "dvc-ssh";
-  version = "4.1.1";
+  version = "4.2.1";
   pyproject = true;
 
+  disabled = pythonOlder "3.9";
+
   src = fetchPypi {
-    inherit pname version;
-    hash = "sha256-lvC6oAXQR4u7s+11n6NgQExPc9yrq3JAmmXtuOw22tI=";
+    pname = "dvc_ssh";
+    inherit version;
+    hash = "sha256-ld6uaAIA+8lHK/TjKtrjtmGKj5847SBMYYvKKN+MkS4=";
   };
 
   pythonRemoveDeps = [
@@ -23,16 +27,18 @@ buildPythonPackage rec {
     "dvc"
   ];
 
-  nativeBuildInputs = [
-    setuptools-scm
-  ];
+  build-system = [ setuptools-scm ];
 
-  propagatedBuildInputs = [
+  dependencies = [
     bcrypt
     dvc-objects
     sshfs
   ];
 
+  optional-dependencies = {
+    gssapi = [ sshfs ];
+  };
+
   # bcrypt is enabled for sshfs in nixpkgs
   postPatch = ''
     substituteInPlace setup.cfg --replace "sshfs[bcrypt]" "sshfs"
@@ -47,7 +53,7 @@ buildPythonPackage rec {
   # ];
 
   meta = with lib; {
-    description = "ssh plugin for dvc";
+    description = "SSH plugin for dvc";
     homepage = "https://pypi.org/project/dvc-ssh/${version}";
     changelog = "https://github.com/iterative/dvc-ssh/releases/tag/${version}";
     license = licenses.asl20;

pkgs/development/python-modules/dvc-webdav/default.nix

@@ -0,0 +1,46 @@
+{
+  lib,
+  buildPythonPackage,
+  dvc-objects,
+  fetchPypi,
+  pythonOlder,
+  setuptools-scm,
+  setuptools,
+  webdav4,
+}:
+
+buildPythonPackage rec {
+  pname = "dvc-webdav";
+  version = "3.0.0";
+  pyproject = true;
+
+  disabled = pythonOlder "3.9";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-Zefu8uvINBWo3b3LV5vyGaN5fGfnpi1FaMXILeK2pQg=";
+  };
+
+  # Prevent circular dependency
+  pythonRemoveDeps = [ "dvc" ];
+
+  build-system = [
+    setuptools
+    setuptools-scm
+  ];
+
+  dependencies = [
+    dvc-objects
+    webdav4
+  ];
+
+  # Circular dependency
+  # pythonImportsCheck = [ "dvc_webdav" ];
+
+  meta = {
+    description = "Webdav plugin for dvc";
+    homepage = "https://pypi.org/project/dvc-webdav/";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ fab ];
+  };
+}

pkgs/development/python-modules/dvc-webhdfs/default.nix

@@ -0,0 +1,51 @@
+{
+  lib,
+  buildPythonPackage,
+  dvc-objects,
+  fetchPypi,
+  fsspec,
+  pythonOlder,
+  requests-kerberos,
+  setuptools-scm,
+  setuptools,
+}:
+
+buildPythonPackage rec {
+  pname = "dvc-webhdfs";
+  version = "3.1.0";
+  pyproject = true;
+
+  disabled = pythonOlder "3.9";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-bolIQ9Fc52agXGFt7anZvDYSSOk7+eozi5lublHqD+o=";
+  };
+
+  # Prevent circular dependency
+  pythonRemoveDeps = [ "dvc" ];
+
+  build-system = [
+    setuptools
+    setuptools-scm
+  ];
+
+  dependencies = [
+    dvc-objects
+    fsspec
+  ];
+
+  optional-dependencies = {
+    kerberos = [ requests-kerberos ];
+  };
+
+  # Circular dependency
+  # pythonImportsCheck = [ "dvc_webhdfs" ];
+
+  meta = {
+    description = "Webhdfs plugin for dvc";
+    homepage = "https://pypi.org/project/dvc-webhdfs/";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ fab ];
+  };
+}

pkgs/development/python-modules/dvc/default.nix

@@ -1,9 +1,11 @@
 {
   lib,
-  appdirs,
+  attrs,
   buildPythonPackage,
+  celery,
   colorama,
   configobj,
+  dulwich,
   distro,
   dpath,
   dvc-azure,
@@ -12,14 +14,18 @@
   dvc-gs,
   dvc-hdfs,
   dvc-http,
+  dvc-oss,
   dvc-render,
   dvc-s3,
   dvc-ssh,
   dvc-studio-client,
   dvc-task,
+  dvc-webdav,
+  dvc-webhdfs,
   fetchFromGitHub,
   flatten-dict,
   flufl-lock,
+  fsspec,
   funcy,
   grandalf,
   gto,
@@ -27,7 +33,9 @@
   importlib-metadata,
   importlib-resources,
   iterative-telemetry,
+  kombu,
   networkx,
+  omegaconf,
   packaging,
   pathspec,
   platformdirs,
@@ -60,7 +68,7 @@ buildPythonPackage rec {
   version = "3.59.1";
   pyproject = true;
 
-  disabled = pythonOlder "3.8";
+  disabled = pythonOlder "3.9";
 
   src = fetchFromGitHub {
     owner = "iterative";
@@ -85,11 +93,13 @@ buildPythonPackage rec {
 
   dependencies =
     [
-      appdirs
+      attrs
+      celery
       colorama
       configobj
       distro
       dpath
+      dulwich
       dvc-data
       dvc-http
       dvc-render
@@ -97,12 +107,15 @@ buildPythonPackage rec {
       dvc-task
       flatten-dict
       flufl-lock
+      fsspec
       funcy
       grandalf
       gto
       hydra-core
       iterative-telemetry
+      kombu
       networkx
+      omegaconf
       packaging
       pathspec
       platformdirs
@@ -126,17 +139,20 @@ buildPythonPackage rec {
     ++ lib.optionals enableGoogle optional-dependencies.gs
     ++ lib.optionals enableAWS optional-dependencies.s3
     ++ lib.optionals enableAzure optional-dependencies.azure
-    ++ lib.optionals enableSSH optional-dependencies.ssh
-    ++ lib.optionals (pythonOlder "3.8") [ importlib-metadata ]
-    ++ lib.optionals (pythonOlder "3.9") [ importlib-resources ];
+    ++ lib.optionals enableSSH optional-dependencies.ssh;
 
   optional-dependencies = {
     azure = [ dvc-azure ];
     gdrive = [ dvc-gdrive ];
     gs = [ dvc-gs ];
     hdfs = [ dvc-hdfs ];
+    oss = [ dvc-oss ];
     s3 = [ dvc-s3 ];
     ssh = [ dvc-ssh ];
+    ssh_gssapi = [ dvc-ssh ] ++ dvc-ssh.optional-dependencies.gssapi;
+    webdav = [ dvc-webdav ];
+    webhdfs = [ dvc-webhdfs ];
+    webhdfs_kerberos = [ dvc-webhdfs ] ++ dvc-webhdfs.optional-dependencies.kerberos;
   };
 
   # Tests require access to real cloud services

pkgs/top-level/python-packages.nix

@@ -4042,6 +4042,8 @@ self: super: with self; {
 
   dvc-objects = callPackage ../development/python-modules/dvc-objects {  };
 
+  dvc-oss = callPackage ../development/python-modules/dvc-oss {  };
+
   dvc-render = callPackage ../development/python-modules/dvc-render {  };
 
   dvc-s3 = callPackage ../development/python-modules/dvc-s3 { };
@@ -4052,6 +4054,10 @@ self: super: with self; {
 
   dvc-task = callPackage ../development/python-modules/dvc-task {  };
 
+  dvc-webdav = callPackage ../development/python-modules/dvc-webdav {  };
+
+  dvc-webhdfs = callPackage ../development/python-modules/dvc-webhdfs {  };
+
   dvclive = callPackage ../development/python-modules/dvclive {  };
 
   dwdwfsapi = callPackage ../development/python-modules/dwdwfsapi { };