Makes NO_PROXY, HTTP_PROXY and HTTPS_PROXY values configurable in settings

src/manage/commands.py

@@ -256,6 +256,20 @@ def execute(self):
     # Typically configured to '%VIRTUAL_ENV%' to pick up the active environment
     "virtual_env": (str, None, "env", "path"),
 
+    # Overrides for the OS proxy configuration, typically set to '%HTTP_PROXY%'
+    # by default to pick up environment settings, but can be overridden to empty
+    # to disable use of the process environment.
+    "http_proxy": (str, None, "env"),
+    "https_proxy": (str, None, "env"),
+    "no_proxy": (str, None, "env"),
+
+    # Global auth for accessing sources, typically set to '%PYMANAGER_USERNAME%'
+    # and '%PYMANAGER_PASSWORD%'. These should really have been
+    # '%PYTHON_MANAGER_USERNAME%' rather than '%PYMANAGER' - the latter is meant
+    # for debugging/hidden options, not user-visible ones.
+    "username": (str, None, "env"),
+    "password": (str, None, "env"),
+
     "list": {
         "format": (str, None, "env"),
         "unmanaged": (config_bool, None, "env"),
@@ -355,6 +369,9 @@ class BaseCommand:
     default_platform = None
     automatic_install = True
     include_unmanaged = True
+    http_proxy = None
+    https_proxy = None
+    no_proxy = None
     virtual_env = None
     shebang_can_run_anything = True
     shebang_can_run_anything_silently = False

src/manage/install_command.py

@@ -15,7 +15,6 @@
 from .tagutils import install_matches_any, tag_or_range
 from .urlutils import (
     sanitise_url,
-    urlopen as _urlopen,
     urlretrieve as _urlretrieve,
     IndexDownloader,
 )
@@ -48,7 +47,7 @@ def _expand_versions_by_tag(versions):
                 yield {**v, "tag": t}
 
 
-def select_package(index_downloader, tag, platform=None, *, urlopen=_urlopen, by_id=False, allow_pre=True):
+def select_package(index_downloader, tag, platform=None, *, by_id=False, allow_pre=True):
     """Finds suitable package from index.json that looks like:
     {"versions": [
       {"id": ..., "company": ..., "tag": ..., "url": ..., "hash": {"sha256": hexdigest}},
@@ -96,7 +95,7 @@ def select_package(index_downloader, tag, platform=None, *, urlopen=_urlopen, by
     raise RuntimeError("End of select_package reached")
 
 
-def download_package(cmd, install, dest, cache, *, on_progress=None, urlopen=_urlopen, urlretrieve=_urlretrieve):
+def download_package(cmd, install, dest, cache, *, on_progress=None, urlretrieve=_urlretrieve):
     LOGGER.debug("Starting download package %s to %s", sanitise_url(install["url"]), dest)
 
     if not cmd.force and dest.is_file():
@@ -130,7 +129,7 @@ def _find_creds(url):
         if auth:
             LOGGER.debug("Found credentials in URL or configured source.")
             return auth
-        auth = os.getenv("PYMANAGER_USERNAME", ""), os.getenv("PYMANAGER_PASSWORD", "")
+        auth = cmd.username, (cmd.password or "")
         if auth[0]:
             LOGGER.debug("Found credentials in environment.")
             return auth

src/manage/urlutils.py

@@ -102,21 +102,32 @@ def _parse_proxy(value):
     return proxy, credentials
 
 
-def _proxy_settings_from_env():
-    if os.getenv("NO_PROXY"):
-        return _ProxySettings(mode=PROXY_MODE_DIRECT)
+def _proxy_settings(info):
+    if not info:
+        return _ProxySettings()
 
-    http_value = os.getenv("HTTP_PROXY")
-    https_value = os.getenv("HTTPS_PROXY")
-    if not http_value and not https_value:
+    if info.no_proxy:
+        if info.no_proxy.lower() in ("1", "*", "y", "yes", "t", "true"):
+            return _ProxySettings(mode=PROXY_MODE_DIRECT)
+        # TODO: Parse and filter based on info.no_proxy IF we can be sure it's
+        # secure, preferably by delegating responsibility for parsing to the OS.
+        # Claiming to support proper whitelisting of domains is very likely to
+        # attract reports of bypasses, none of which we would treat as a
+        # vulnerability (due to non-default configuration settings), but they
+        # historically create more noise than they're worth. Let's just avoid
+        # breaking into jail until we're sure we can do it right.
+        LOGGER.warn("Values for NO_PROXY other than 1, *, y[es] and t[rue] are "
+            "not supported. Other proxy settings will be used.")
+
+    if not info.http_proxy and not info.https_proxy:
         return _ProxySettings()
 
     http_proxy = http_credentials = None
     https_proxy = https_credentials = None
-    if http_value:
-        http_proxy, http_credentials = _parse_proxy(http_value)
-    if https_value:
-        https_proxy, https_credentials = _parse_proxy(https_value)
+    if info.http_proxy:
+        http_proxy, http_credentials = _parse_proxy(info.http_proxy)
+    if info.https_proxy:
+        https_proxy, https_credentials = _parse_proxy(info.https_proxy)
 
     proxy_list = " ".join(
         value for value in (
@@ -134,20 +145,27 @@ def _proxy_settings_from_env():
     )
 
 
+class _EnvProxyInfo:
+    """Default value for proxy_info arguments from environment variables."""
+    no_proxy = os.getenv("NO_PROXY")
+    http_proxy = os.getenv("HTTP_PROXY")
+    https_proxy = os.getenv("HTTPS_PROXY")
+
+
 class NoInternetError(Exception):
     pass
 
 
 class _Request:
-    def __init__(self, url, method="GET", headers={}, outfile=None):
+    def __init__(self, url, method="GET", headers={}, outfile=None, proxy_info=None):
         self.url = url
         self.method = method.upper()
         self.headers = dict(headers)
         self.chunksize = 64 * 1024
         self.username = None
         self.password = None
         self.outfile = Path(outfile) if outfile else None
-        self.proxy_settings = _proxy_settings_from_env()
+        self.proxy_settings = _proxy_settings(proxy_info)
         self._on_progress = None
         self._on_auth_request = None
         self._on_cancel = None
@@ -488,7 +506,8 @@ def _powershell_urlretrieve(request):
                 raise
 
 
-def urlopen(url, method="GET", headers={}, on_progress=None, on_auth_request=None):
+def urlopen(url, method="GET", headers={}, on_progress=None,
+            on_auth_request=None, proxy_info=_EnvProxyInfo):
     scheme, sep, path = url.partition("://")
     if not sep:
         scheme = "file"
@@ -500,7 +519,7 @@ def urlopen(url, method="GET", headers={}, on_progress=None, on_auth_request=Non
         with open(file_url_to_path(url), "rb") as f:
             return f.read()
 
-    request = _Request(url, method=method, headers=headers)
+    request = _Request(url, method=method, headers=headers, proxy_info=proxy_info)
     request._on_progress = on_progress
     request._on_auth_request = on_auth_request
 
@@ -561,7 +580,8 @@ def urlopen(url, method="GET", headers={}, on_progress=None, on_auth_request=Non
 
 
 def urlretrieve(url, outfile, method="GET", headers={}, chunksize=64 * 1024,
-                on_progress=None, on_auth_request=None, on_cancel=None):
+                on_progress=None, on_auth_request=None, on_cancel=None,
+                proxy_info=_EnvProxyInfo):
     scheme, sep, path = url.partition("://")
     if not sep:
         scheme = "file"
@@ -588,7 +608,7 @@ def on_progress(_): pass
             on_progress(100)
         return
 
-    request = _Request(url, method=method, headers=headers)
+    request = _Request(url, method=method, headers=headers, proxy_info=proxy_info)
     request.outfile = Path(outfile)
     request.chunksize = chunksize
     request._on_progress = on_progress

src/pymanager.json

@@ -12,6 +12,14 @@
     "automatic_install": "%PYTHON_MANAGER_AUTOMATIC_INSTALL%",
     "include_unmanaged": "%PYTHON_MANAGER_INCLUDE_UNMANAGED%",
     "virtual_env": "%VIRTUAL_ENV%",
+
+    "http_proxy": "%HTTP_PROXY%",
+    "https_proxy": "%HTTPS_PROXY%",
+    "no_proxy": "%NO_PROXY",
+
+    "username": "%PYMANAGER_USERNAME%",
+    "password": "%PYMANAGER_PASSWORD%",
+
     "shebang_can_run_anything": "%PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING%",
     "shebang_can_run_anything_silently": false,