diff --git a/pkgs/development/python-modules/lm-eval/default.nix b/pkgs/development/python-modules/lm-eval/default.nix
new file mode 100644
index 000000000000..87e71e4d3fc8
--- /dev/null
+++ b/pkgs/development/python-modules/lm-eval/default.nix
@@ -0,0 +1,156 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  setuptools-scm,
+  accelerate,
+  aiohttp,
+  antlr4-python3-runtime,
+  causal-conv1d,
+  datasets,
+  dill,
+  evaluate,
+  hf-transfer,
+  immutabledict,
+  jsonlines,
+  langdetect,
+  mamba-ssm,
+  more-itertools,
+  nltk,
+  numexpr,
+  numpy,
+  optimum,
+  pandas,
+  peft,
+  pybind11,
+  pytablewriter,
+  pytestCheckHook,
+  requests,
+  rouge-score,
+  sacrebleu,
+  scikit-learn,
+  sentencepiece,
+  sqlitedict,
+  sympy,
+  tenacity,
+  tiktoken,
+  torch,
+  tqdm,
+  tqdm-multiprocess,
+  transformers,
+  vllm,
+  wandb,
+  word2number,
+  zstandard,
+}:
+
+buildPythonPackage rec {
+  pname = "lm-eval";
+  version = "0.4.7";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "EleutherAI";
+    repo = "lm-evaluation-harness";
+    tag = "v${version}";
+    hash = "sha256-UEY0KrsgWoyj0fnKBca2yYjVTBFje6DdLfvnNFVYFHU=";
+  };
+
+  build-system = [
+    setuptools-scm
+  ];
+
+  dependencies = [
+    accelerate
+    datasets
+    dill
+    evaluate
+    jsonlines
+    more-itertools
+    numexpr
+    peft
+    pybind11
+    pytablewriter
+    rouge-score
+    sacrebleu
+    scikit-learn
+    sqlitedict
+    torch
+    tqdm-multiprocess
+    transformers
+    word2number
+    zstandard
+  ];
+
+  optional-dependencies = {
+    api = [
+      requests
+      aiohttp
+      tenacity
+      tqdm
+      tiktoken
+    ];
+    hf_transfer = [ hf-transfer ];
+    ifeval = [
+      langdetect
+      immutabledict
+      nltk
+    ];
+    neuronx = [ optimum ] ++ optimum.optional-dependencies.neuronx;
+    mamba = [
+      mamba-ssm
+      causal-conv1d
+    ];
+    math = [
+      sympy
+      antlr4-python3-runtime
+    ];
+    optimum = [ optimum ] ++ optimum.optional-dependencies.openvino;
+    sentencepiece = [ sentencepiece ];
+    vllm = [ vllm ];
+    wandb = [
+      wandb
+      pandas
+      numpy
+    ];
+    # Still missing dependencies for the following:
+    # deepsparse, gptq, ibm_watsonx_ai, multilingual, promptsource, sparseml,
+    # zeno, gptqmodel, japanese_leaderboard; all = [...];
+  };
+
+  pythonImportsCheck = [ "lm_eval" ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ] ++ optional-dependencies.api;
+
+  preCheck = ''
+    export HOME=$TMP
+  '';
+
+  disabledTests = [
+    "test_deepsparse" # deepsparse is not available
+    "test_model_tokenized_call_usage" # downloads a model
+  ];
+
+  disabledTestPaths = [
+    # attempts to download models
+    "tests/models/test_huggingface.py"
+    "tests/test_evaluator.py"
+    "tests/test_include_path.py"
+    "tests/test_prompt.py"
+    "tests/test_task_manager.py"
+    "tests/test_tasks.py"
+
+    # optimum-intel is not available
+    "tests/models/test_openvino.py"
+  ];
+
+  meta = {
+    changelog = "https://github.com/EleutherAI/lm-evaluation-harness/releases/tag/v${version}";
+    description = "A framework for few-shot evaluation of language models";
+    homepage = "https://github.com/EleutherAI/lm-evaluation-harness";
+    license = [ lib.licenses.mit ];
+    maintainers = [ lib.maintainers.booxter ];
+  };
+}
diff --git a/pkgs/development/python-modules/tqdm-multiprocess/default.nix b/pkgs/development/python-modules/tqdm-multiprocess/default.nix
new file mode 100644
index 000000000000..812a694078d0
--- /dev/null
+++ b/pkgs/development/python-modules/tqdm-multiprocess/default.nix
@@ -0,0 +1,41 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  setuptools-scm,
+  tqdm,
+  colorama,
+}:
+
+buildPythonPackage {
+  pname = "tqdm-multiprocess";
+  version = "0.0.11";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "EleutherAI";
+    repo = "tqdm-multiprocess";
+    rev = "fccefc473595055bf3a5e74bcf8a75b3a9517638";
+    hash = "sha256-nQeFPwF5OasOYrVs7kLG/Uz6pf1FKxar4ygggo8s4ZM=";
+  };
+
+  build-system = [
+    setuptools-scm
+  ];
+
+  dependencies = [
+    colorama
+    tqdm
+  ];
+
+  pythonImportsCheck = [
+    "tqdm_multiprocess"
+  ];
+
+  meta = {
+    description = "Support multiple worker processes, each with multiple tqdm progress bars, displaying them cleanly through the main process";
+    homepage = "https://github.com/EleutherAI/tqdm-multiprocess";
+    license = [ lib.licenses.mit ];
+    maintainers = [ lib.maintainers.booxter ];
+  };
+}
diff --git a/pkgs/development/python-modules/word2number/default.nix b/pkgs/development/python-modules/word2number/default.nix
new file mode 100644
index 000000000000..fbfc25cf889f
--- /dev/null
+++ b/pkgs/development/python-modules/word2number/default.nix
@@ -0,0 +1,45 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  setuptools-scm,
+  future,
+  python,
+}:
+
+buildPythonPackage rec {
+  pname = "word2number";
+  version = "1.1";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "akshaynagpal";
+    repo = "w2n";
+    tag = version;
+    hash = "sha256-dgHPEfieNDZnP6+YvywvN3ZzmeICav0WMYKkWDSJ/LE=";
+  };
+
+  build-system = [
+    setuptools-scm
+  ];
+
+  dependencies = [
+    future
+  ];
+
+  pythonImportsCheck = [
+    "word2number"
+  ];
+
+  checkPhase = ''
+    ${lib.getExe python} unit_testing.py
+  '';
+
+  meta = {
+    changelog = "https://github.com/akshaynagpal/w2n/releases/tag/${version}";
+    description = "Convert number words (eg. twenty one) to numeric digits (21)";
+    homepage = "http://w2n.readthedocs.io/";
+    license = [ lib.licenses.mit ];
+    maintainers = [ lib.maintainers.booxter ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 7375457cbb86..7294f354a1cd 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -7657,6 +7657,8 @@ self: super: with self; {
     llvm = pkgs.llvm_14;
   };
 
+  lm-eval = callPackage ../development/python-modules/lm-eval { };
+
   lm-format-enforcer = callPackage ../development/python-modules/lm-format-enforcer { };
 
   lmdb = callPackage ../development/python-modules/lmdb {
@@ -16451,6 +16453,8 @@ self: super: with self; {
 
   tqdm = callPackage ../development/python-modules/tqdm { };
 
+  tqdm-multiprocess = callPackage ../development/python-modules/tqdm-multiprocess { };
+
   traceback2 = callPackage ../development/python-modules/traceback2 { };
 
   tracerite = callPackage ../development/python-modules/tracerite { };
@@ -18204,6 +18208,8 @@ self: super: with self; {
 
   woodblock = callPackage ../development/python-modules/woodblock { };
 
+  word2number = callPackage ../development/python-modules/word2number { };
+
   wordcloud = callPackage ../development/python-modules/wordcloud { };
 
   wordfreq = callPackage ../development/python-modules/wordfreq { };