Last active
July 18, 2021 11:51
-
-
Save decatur/fce3cbf8518149e9597887ca10163266 to your computer and use it in GitHub Desktop.
Way out of Python venv hell.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
So you are doing Python on MS Windows. Maybe even deploying your stuff on a Windows Server. | |
By now you know that virtual environments are HEAVY on Windows, and you have many projects or application with the same | |
copy of numpy (60MB each) over and over again. | |
No more. Install a package by unpacking wheels (not pip install) to the corresponding versioned folder: | |
__pypackages__ | |
└── numpy-1.21.0 | |
└── numpy | |
└── numpy-1.21.0.dist-info | |
└── dateutil-2.8.1 | |
└── dateutil | |
│ └── parser | |
│ └── tz | |
│ └── zoneinfo | |
└── python_dateutil-2.8.1.dist-info | |
This differs from https://www.python.org/dev/peps/pep-0582, which _locally_ caches packages by _python version_. | |
See also https://discuss.python.org/t/pep-582-python-local-packages-directory/963 | |
Remarks: | |
IDE support for finding packages is nil, as VsCode or PyCharm will look into a **fixed** set of configurable folders. | |
Would symlinks help, or oblitarate this approach in the first place? | |
Yes it does (both Python and PyCharm) on MS Windows, if you have the right to create symbolic links: | |
set PROJECT_DIR=SOMETHING | |
mklink /d %PROJECT_DIR%\venv\Lib\site-packages\dateutil C:\__pypackages__\python_dateutil-2.8.1\dateutil | |
So we could create symbolic links from distributions_by_package_name in site-dir if allowed (most likely on PC), | |
else use VersionFinder (on Server). | |
""" | |
import importlib.util | |
import sys | |
from importlib.machinery import SourceFileLoader | |
import importlib.metadata | |
from pathlib import Path | |
from typing import Dict | |
class VersionFinder(importlib.machinery.PathFinder): | |
path_by_name: Dict[str, Path] = dict() | |
@classmethod | |
def find_spec(cls, name: str, path, target=None): | |
# print(f"Importing {name} {path}") | |
if name in cls.path_by_name: | |
path = [cls.path_by_name[name].as_posix()] | |
return super().find_spec(name, path) | |
@classmethod | |
def find_distributions(cls, ctx): | |
if ctx.name in cls.path_by_name: | |
dist_info = next(cls.path_by_name[ctx.name].glob('*.dist-info')) | |
return iter([importlib.metadata.PathDistribution(dist_info)]) | |
else: | |
return super().find_distributions(ctx) | |
def simple_requirements_parser(requirements: str) -> Dict[str, Path]: | |
distributions_by_package_name = dict() | |
for line in requirements.splitlines(): | |
line = line.strip() | |
if not line: | |
continue | |
name, version = line.split('==') | |
name = name.replace('-', '_').lower() | |
dist_folder = Path('__pypackages__') / f'{name}-{version}' | |
if not dist_folder.is_dir(): | |
print(f'Cannot find distribution {dist_folder.as_posix()}', file=sys.stderr) | |
continue | |
for item in dist_folder.iterdir(): | |
if item.is_dir() and item.suffix != '.dist-info': | |
distributions_by_package_name[item.name] = dist_folder | |
return distributions_by_package_name | |
# Fill this from your fixed requirements.txt. I recommend pip-compile strongly! | |
VersionFinder.path_by_name = simple_requirements_parser(""" | |
numpy==1.21.0 | |
python-dateutil==2.8.1 | |
""") | |
sys.meta_path.insert(0, VersionFinder) | |
import dateutil.parser | |
print(dateutil.parser.parse('2021-01-01')) | |
print(importlib.metadata.version('dateutil')) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment