binary-devops · aditya-deriv · Oct 25, 2019
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,17 @@
 # Poise-Python Changelog
 
+## v1.7.1
+
+* Support for Pip 18.1.
+* Improved support for Python 3 on Ubuntu 18.04.
+* Update SCL packages.
+
+## v1.7.0
+
+* Support for Pip 10.
+* Support for Chef 14.
+* System package info for Ubuntu 18.04 and Debian 9, 10.
+
 ## v1.6.0
 
 * Improved handling for Python 3.3.

diff --git a/Gemfile b/Gemfile
@@ -27,7 +27,7 @@ def dev_gem(name, path: File.join('..', name), github: nil)
   end
 end
 
-dev_gem 'halite'
+dev_gem 'halite', github: 'poise/halite'
 dev_gem 'poise'
 dev_gem 'poise-archive'
 dev_gem 'poise-boiler'

diff --git a/Rakefile b/Rakefile
@@ -14,4 +14,5 @@
 # limitations under the License.
 #
 
+ENV['KITCHEN_CONCURRENCY'] = '10'
 require 'poise_boiler/rakefile'
diff --git a/lib/poise_python/python_providers/scl.rb b/lib/poise_python/python_providers/scl.rb
@@ -26,7 +26,8 @@ module PythonProviders
     class Scl < Base
       include PoiseLanguages::Scl::Mixin
       provides(:scl)
-      scl_package('3.5.1', 'rh-python35', 'rh-python35-python-devel', '>= 7.0')
+      scl_package('3.6.3', 'rh-python36', 'rh-python36-python-devel')
+      scl_package('3.5.1', 'rh-python35', 'rh-python35-python-devel')
       scl_package('3.4.2', 'rh-python34', 'rh-python34-python-devel')
       scl_package('3.3.2', 'python33', 'python33-python-devel')
       scl_package('2.7.8', 'python27', 'python27-python-devel')

diff --git a/lib/poise_python/python_providers/system.rb b/lib/poise_python/python_providers/system.rb
@@ -28,11 +28,14 @@ class System < Base
       provides(:system)
       packages('python', {
         debian: {
+          '~> 10.0' => %w{python3.6 python2.7},
+          '~> 9.0' => %w{python3.5 python2.7},
           '~> 8.0' => %w{python3.4 python2.7},
           '~> 7.0' => %w{python3.2 python2.7 python2.6},
           '~> 6.0' => %w{python3.1 python2.6 python2.5},
         },
         ubuntu: {
+          '18.04' => %w{python3.6 python2.7},
           '16.04' => %w{python3.5 python2.7},
           '14.04' => %w{python3.4 python2.7},
           '12.04' => %w{python3.2 python2.7},
@@ -54,9 +57,22 @@ def python_binary
 
       def install_python
         install_system_packages
+        if node.platform_family?('debian') && system_package_name == 'python3.6'
+          # Ubuntu 18.04 and Debian 10 have some weird dependency fuckery going on.
+          _options = options
+          package %w{python3.6-venv python3.6-distutils} do
+            action(:upgrade) if _options['package_upgrade']
+          end
+        end
       end
 
       def uninstall_python
+        if node.platform_family?('debian') && system_package_name == 'python3.6'
+          # Other side of the depdency nonsense.
+          package %w{python3.6-venv python3.6-distutils} do
+            action(:purge)
+          end
+        end
         uninstall_system_packages
       end
 
@@ -71,7 +87,7 @@ def system_package_candidates(version)
           end
           # Aliases for 2 and 3.
           if version == '3' || version == ''
-            names.concat(%w{python3.5 python35 python3.4 python34 python3.3 python33 python3.2 python32 python3.1 python31 python3.0 python30 python3})
+            names.concat(%w{python3.7 python37 python3.6 python36 python3.5 python35 python3.4 python34 python3.3 python33 python3.2 python32 python3.1 python31 python3.0 python30 python3})
           end
           if version == '2' || version == ''
             names.concat(%w{python2.7 python27 python2.6 python26 python2.5 python25})

diff --git a/lib/poise_python/resources/python_package.rb b/lib/poise_python/resources/python_package.rb
@@ -38,14 +38,25 @@ module PythonPackage
 
 import pip
 # Don't use pkg_resources because I don't want to require it before this anyway.
-if re.match(r'0|1|6\\.0', pip.__version__):
+if re.match(r'0\\.|1\\.|6\\.0', pip.__version__):
   sys.stderr.write('The python_package resource requires pip >= 6.1.0, currently '+pip.__version__+'\\n')
   sys.exit(1)
 
-from pip.commands import InstallCommand
-from pip.index import PackageFinder
-from pip.req import InstallRequirement
-
+try:
+  from pip.commands import InstallCommand
+  from pip.index import PackageFinder
+  from pip.req import InstallRequirement
+  install_req_from_line = InstallRequirement.from_line
+except ImportError:
+  # Pip 10 moved all internals to their own package.
+  from pip._internal.commands import InstallCommand
+  from pip._internal.index import PackageFinder
+  try:
+    # Pip 18.1 moved from_line to the constructors
+    from pip._internal.req.constructors import install_req_from_line
+  except ImportError:
+    from pip._internal.req import InstallRequirement
+    install_req_from_line = InstallRequirement.from_line
 
 packages = {}
 cmd = InstallCommand()
@@ -68,7 +79,7 @@ module PythonPackage
   finder = PackageFinder(**finder_options)
   find_all = getattr(finder, 'find_all_candidates', getattr(finder, '_find_all_versions', None))
   for arg in args:
-    req = InstallRequirement.from_line(arg)
+    req = install_req_from_line(arg)
     found = finder.find_requirement(req, True)
     all_candidates = find_all(req.name)
     candidate = [c for c in all_candidates if c.location == found]
@@ -113,6 +124,12 @@ class Resource < Chef::Resource::Package
         #   @return [String, Integer, nil]
         attribute(:user, kind_of: [String, Integer, NilClass], default: lazy { default_user })
 
+        # This should probably be in the base class but ¯\_(ツ)_/¯.
+        # @!attribute allow_downgrade
+        #   Allow downgrading the package.
+        #   @return [Boolean]
+        attribute(:allow_downgrade, kind_of: [TrueClass, FalseClass], default: false)
+
         def initialize(*args)
           super
           # For older Chef.
@@ -187,6 +204,7 @@ def load_current_resource
           @current_resource = new_resource.class.new(new_resource.name, run_context)
           current_resource.package_name(new_resource.package_name)
           check_package_versions(current_resource)
+          Chef::Log.debug("[#{new_resource}] Current version: #{current_resource.version}, candidate version: #{@candidate_version}")
           current_resource
         end
 
@@ -200,7 +218,7 @@ def load_current_resource
         def check_package_versions(resource, version=new_resource.version)
           version_data = Hash.new {|hash, key| hash[key] = {current: nil, candidate: nil} }
           # Get the version for everything currently installed.
-          list = pip_command('list', :list).stdout
+          list = pip_command('list', :list, [], environment: {'PIP_FORMAT' => 'json'}).stdout
           parse_pip_list(list).each do |name, current|
             # Merge current versions in to the data.
             version_data[name][:current] = current
@@ -347,15 +365,22 @@ def pip_outdated(requirements)
         # @param text [String] Output to parse.
         # @return [Hash<String, String>]
         def parse_pip_list(text)
-          text.split(/\r?\n/).inject({}) do |memo, line|
-            # Example of a line:
-            # boto (2.25.0)
-            if md = line.match(/^(\S+)\s+\(([^\s,]+).*\)$/i)
-              memo[parse_package_name(md[1])] = md[2]
-            else
-              Chef::Log.debug("[#{new_resource}] Unparsable line in pip list: #{line}")
+          if text[0] == '['
+            # Pip 9 or newer, so it understood $PIP_FORMAT=json.
+            Chef::JSONCompat.parse(text).each_with_object({}) do |data, memo|
+              memo[parse_package_name(data['name'])] = data['version']
+            end
+          else
+            # Pip 8 or earlier, which doesn't support JSON output.
+            text.split(/\r?\n/).each_with_object({}) do |line, memo|
+              # Example of a line:
+              # boto (2.25.0)
+              if md = line.match(/^(\S+)\s+\(([^\s,]+).*\)$/i)
+                memo[parse_package_name(md[1])] = md[2]
+              else
+                Chef::Log.debug("[#{new_resource}] Unparsable line in pip list: #{line}")
+              end
             end
-            memo
           end
         end
 

diff --git a/lib/poise_python/resources/python_runtime.rb b/lib/poise_python/resources/python_runtime.rb
@@ -51,6 +51,7 @@ class Resource < Chef::Resource
         #   of https://bootstrap.pypa.io/get-pip.py is used. If you want to skip
         #   the pip installer entirely, set {#pip_version} to `false`.
         #   @return [String]
+        # If this default value changes, fix ths 2.6-compat logic in python_runtime_pip.
         attribute(:get_pip_url, kind_of: String, default: 'https://bootstrap.pypa.io/get-pip.py')
         # @!attribute pip_version
         #   Version of pip to install. If set to `true`, the latest available

diff --git a/lib/poise_python/resources/python_runtime_pip.rb b/lib/poise_python/resources/python_runtime_pip.rb
@@ -30,6 +30,12 @@ module PythonRuntimePip
       # Earliest version of pip we will try upgrading in-place.
       PIP_INPLACE_VERSION = Gem::Version.create('7.0.0')
 
+      # Version to trigger the automatic get-pip.py URL fixup on for 2.6 compat.
+      PY26_FIXUP_VERSION = Gem::Version.create('2.7')
+
+      # Replacement URL for 2.6 compat.
+      PY26_FIXUP_GETPIP_URL = 'https://bootstrap.pypa.io/2.6/get-pip.py'
+
       # A `python_runtime_pip` resource to install/upgrade pip itself. This is
       # used internally by `python_runtime` and is not intended to be a public
       # API.
@@ -103,15 +109,30 @@ def action_uninstall
         #
         # @return [void]
         def bootstrap_pip
+          # If we're on Python 2.6 and using the default get_pip_url, we need to
+          # switch to a 2.6-compatible version. This kind of sucks because it
+          # makes the default a magic value but it seems like the safest option.
+          get_pip_url = new_resource.get_pip_url
+          if get_pip_url == 'https://bootstrap.pypa.io/get-pip.py'
+            python_version_cmd = poise_shell_out!([new_resource.parent.python_binary, '--version'], environment: new_resource.parent.python_environment)
+            # Python 2 puts the output on stderr, 3 is on stdout. You can't make this shit up.
+            python_version = (python_version_cmd.stdout + python_version_cmd.stderr)[/Python (\S+)/, 1]
+            Chef::Log.debug("[#{new_resource}] Checking for Python 2.6 fixup of get-pip URL, found Python version #{python_version || '(unknown)'}")
+            if python_version && Gem::Version.create(python_version) < PY26_FIXUP_VERSION
+              Chef::Log.debug("[#{new_resource}] Detected old Python, enabling fixup")
+              get_pip_url = PY26_FIXUP_GETPIP_URL
+            end
+          end
+
           # Always updated if we have hit this point.
-          converge_by("Bootstrapping pip #{new_resource.version || 'latest'} from #{new_resource.get_pip_url}") do
+          converge_by("Bootstrapping pip #{new_resource.version || 'latest'} from #{get_pip_url}") do
             # Use a temp file to hold the installer.
             # Put `Tempfile.create` back when Chef on Windows has a newer Ruby.
             # Tempfile.create(['get-pip', '.py']) do |temp|
             temp = Tempfile.new(['get-pip', '.py'])
             begin
               # Download the get-pip.py.
-              get_pip = Chef::HTTP.new(new_resource.get_pip_url).get('')
+              get_pip = Chef::HTTP.new(get_pip_url).get('')
               # Write it to the temp file.
               temp.write(get_pip)
               # Close the file to flush it.
@@ -165,6 +186,7 @@ def install_pip
               action :upgrade
               parent_python new_resource.parent
               version new_resource.version if new_resource.version
+              allow_downgrade true
             end
           end
         end

diff --git a/lib/poise_python/resources/python_runtime_test.rb b/lib/poise_python/resources/python_runtime_test.rb
@@ -70,6 +70,10 @@ def action_run
             test_version
 
             # Test python_package.
+            python_package 'argparse' do
+              # Needed for sqlparse but not in the stdlib until 2.7.
+              python new_resource.name
+            end
             python_package 'sqlparse remove before' do
               action :remove
               package_name 'sqlparse'
@@ -176,6 +180,7 @@ def action_run
               end
               test_import('docopt', python: nil, virtualenv: test_venv, user: test_user)
             end
+
           end
         end
 

diff --git a/lib/poise_python/version.rb b/lib/poise_python/version.rb
@@ -16,5 +16,5 @@
 
 
 module PoisePython
-  VERSION = '1.6.1.pre'
+  VERSION = '1.7.1.pre'
 end
diff --git a/poise-python.gemspec b/poise-python.gemspec
@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = %w{lib}
 
-  spec.add_dependency 'chef', '>= 12.1', '< 14'
+  spec.add_dependency 'chef', '>= 12.16', '< 15'
   spec.add_dependency 'halite', '~> 1.0'
   spec.add_dependency 'poise', '~> 2.7'
   spec.add_dependency 'poise-languages', '~> 2.0'

diff --git a/test/cookbook/recipes/default.rb b/test/cookbook/recipes/default.rb
@@ -55,24 +55,47 @@
   when 'pypy'
     python_runtime_test 'pypy'
   when 'pip'
-    # Specific test for pip reversion working correctly.
+    # Some pip-related tests that I don't need to run on every Python version.
+    # Pip does plenty of testing on different Python versions and I already touch
+    # the basics.
     pip_provider = value_for_platform_family(default: :portable_pypy, windows: :msi)
+    # Check the baseline state, should pull the latest pip.
     python_runtime 'pip1' do
-      pip_version '7.1.2'
       provider pip_provider
       options path: '/test_pip1'
       version ''
     end
+    # Check installing a requested version.
     python_runtime 'pip2' do
+      pip_version '8.1.2'
       provider pip_provider
       options path: '/test_pip2'
       version ''
     end
-    python_runtime 'pip2b' do
+    # Check installing the latest and reverting to an old version.
+    python_runtime 'pip3' do
+      provider pip_provider
+      options path: '/test_pip3'
+      version ''
+    end
+    python_runtime 'pip3b' do
       pip_version '7.1.2'
       provider pip_provider
-      options path: '/test_pip2'
+      options path: '/test_pip3'
       version ''
     end
+    # Test pip9 specifically just to be safe.
+    python_runtime 'pip4' do
+      pip_version '9.0.3'
+      provider pip_provider
+      options path: '/test_pip4'
+      version ''
+    end
+    # Run a simple package install on each just to test things.
+    (1..4).each do |n|
+      python_package 'structlog' do
+        python "pip#{n}"
+      end
+    end
   end
 end
diff --git a/test/gemfiles/chef-12.1.gemfile b/test/gemfiles/chef-12.1.gemfile
diff --git a/test/gemfiles/chef-12.10.gemfile b/test/gemfiles/chef-12.10.gemfile