Fixing build system

[KOLANICH]

Currently builded package is unusable.
1 version.py doesn't go into a wheel
2 package cannot be called python -m binwalk because of the lack of __main__.py
3 The package is not registered when installed from a wheel.

I have fixed the issues and modernized the build system a bit.

From c76f13df3a185f98176a277ba3b560ddeebfe7f2 Mon Sep 17 00:00:00 2001
From: KOLANICH
Date: Tue, 5 Jun 2018 19:28:01 +0300
Subject: [PATCH] Fixed build

---
 setup.cfg                                    |  35 +++++++
 setup.py                                     | 102 ++++++++++---------
 src/{scripts/binwalk => binwalk/__main__.py} |   0
 3 files changed, 88 insertions(+), 49 deletions(-)
 create mode 100644 setup.cfg
 rename src/{scripts/binwalk => binwalk/__main__.py} (100%)
 mode change 100755 => 100644

diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..8d38ad7
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,35 @@
+[metadata]
+name = binwalk
+author = Craig Heffner
+description = Firmware analysis tool
+long_description = file: README.md
+keywords = firmware, binary, analysis, reverse-engineering, signature
+license = MIT
+classifiers =
+	Programming Language :: Python
+	Programming Language :: Python :: 2
+	Programming Language :: Python :: 3
+	Development Status :: 4 - Beta
+	Environment :: Other Environment
+	Intended Audience :: Developers
+	Intended Audience :: Researchers
+	License :: OSI Approved :: MIT
+	Operating System :: OS Independent
+
+[options]
+zip_safe = True
+include_package_data = True
+
+#test_suite = tests.tests.Tests
+dependency_links = ["git+https://github.com/devttys0/yaffshiv.git"]
+
+[options.extras_require]
+plots = matplotlib
+x86 = capstone
+YAFFS = yaffshiv
+
+[options.entry_points]
+console_scripts = binwalk = binwalk.__main__:main
+
+[build-system]
+requires = ["setuptools", "wheel"]
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 6f44c61..2a741d1 100755
--- a/setup.py
+++ b/setup.py
@@ -1,4 +1,8 @@
 #!/usr/bin/env python
+import os
+from setuptools import setup
+from setuptools.config import read_configuration
+
 import os
 import sys
 import glob
@@ -7,11 +11,18 @@ import subprocess
 from distutils.core import setup, Command
 from distutils.dir_util import remove_tree
 
-MODULE_NAME = "binwalk"
+MODULE_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
+cfg = read_configuration(os.path.join(MODULE_DIRECTORY, 'setup.cfg'))
+cfg["options"].update(cfg["metadata"])
+cfg = cfg["options"]
+
+MODULE_NAME = cfg["name"]
 MODULE_VERSION = "2.1.2"
 SCRIPT_NAME = MODULE_NAME
-MODULE_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
-VERSION_FILE = os.path.join(MODULE_DIRECTORY, "src", "binwalk", "core", "version.py")
+
+PACKAGE_DIR = "src"
+BINWALK_ROOT_DIR = os.path.join(PACKAGE_DIR, MODULE_NAME)
+VERSION_FILE = os.path.join(MODULE_DIRECTORY, BINWALK_ROOT_DIR, "core", "version.py")
 
 # Python3 has a built-in DEVNULL; for Python2, we have to open
 # os.devnull to redirect subprocess stderr output to the ether.
@@ -97,7 +108,7 @@ def remove_binwalk_module(pydir=None, pybin=None):
 
 
 class IDAUnInstallCommand(Command):
-    description = "Uninstalls the binwalk IDA plugin module"
+    description = "Uninstalls the "+MODULE_NAME+" IDA plugin module"
     user_options = [
         ('idadir=', None, 'Specify the path to your IDA install directory.'),
     ]
@@ -117,7 +128,7 @@ class IDAUnInstallCommand(Command):
             return
 
         binida_dst_path = os.path.join(self.idadir, 'plugins', 'binida.py')
-        binwalk_dst_path = os.path.join(self.idadir, 'python', 'binwalk')
+        binwalk_dst_path = os.path.join(self.idadir, 'python', MODULE_NAME)
 
         if os.path.exists(binida_dst_path):
             sys.stdout.write("removing '%s'\n" % binida_dst_path)
@@ -128,7 +139,7 @@ class IDAUnInstallCommand(Command):
 
 
 class IDAInstallCommand(Command):
-    description = "Installs the binwalk IDA plugin module"
+    description = "Installs the "+MODULE_NAME+" IDA plugin module"
     user_options = [
         ('idadir=', None, 'Specify the path to your IDA install directory.'),
     ]
@@ -161,12 +172,12 @@ class IDAInstallCommand(Command):
                 binida_dst_path)
             return
 
-        binwalk_src_path = os.path.join(self.mydir, 'binwalk')
+        binwalk_src_path = os.path.join(self.mydir, MODULE_NAME)
         binwalk_dst_path = os.path.join(self.idadir, 'python')
 
         if not os.path.exists(binwalk_src_path):
             sys.stderr.write(
-                "ERROR: could not locate binwalk source directory '%s'!\n" %
+                "ERROR: could not locate "+MODULE_NAME+" source directory '%s'!\n" %
                 binwalk_src_path)
             return
         if not os.path.exists(binwalk_dst_path):
@@ -176,7 +187,7 @@ class IDAInstallCommand(Command):
             return
 
         binida_dst_path = os.path.join(binida_dst_path, 'binida.py')
-        binwalk_dst_path = os.path.join(binwalk_dst_path, 'binwalk')
+        binwalk_dst_path = os.path.join(binwalk_dst_path, MODULE_NAME)
 
         if os.path.exists(binida_dst_path):
             os.remove(binida_dst_path)
@@ -195,8 +206,8 @@ class IDAInstallCommand(Command):
 class UninstallCommand(Command):
     description = "Uninstalls the Python module"
     user_options = [
-        ('pydir=', None, 'Specify the path to the binwalk python module to be removed.'),
-        ('pybin=', None, 'Specify the path to the binwalk executable to be removed.'),
+        ('pydir=', None, 'Specify the path to the '+MODULE_NAME+' python module to be removed.'),
+        ('pybin=', None, 'Specify the path to the '+MODULE_NAME+' executable to be removed.'),
     ]
 
     def initialize_options(self):
@@ -256,7 +267,7 @@ class AutoCompleteCommand(Command):
     def run(self):
         options = []
         autocomplete_file_path = "/etc/bash_completion.d/%s" % MODULE_NAME
-        auto_complete = '''_binwalk()
+        auto_complete = "_"+MODULE_NAME+'''()
 {
     local cur prev opts
     COMPREPLY=()
@@ -269,9 +280,9 @@ class AutoCompleteCommand(Command):
         return 0
     fi
 }
-complete -F _binwalk binwalk'''
+complete -F _'''+MODULE_NAME+" "+MODULE_NAME
 
-        (out, err) = subprocess.Popen(["binwalk", "--help"], stdout=subprocess.PIPE).communicate()
+        (out, err) = subprocess.Popen([MODULE_NAME, "--help"], stdout=subprocess.PIPE).communicate()
         for line in out.splitlines():
             if b'--' in line:
                 long_opt = line.split(b'--')[1].split(b'=')[0].split()[0].strip()
@@ -323,43 +334,36 @@ class TestCommand(Command):
 
 # The data files to install along with the module
 install_data_files = []
-for data_dir in ["magic", "config", "plugins", "modules", "core"]:
-    install_data_files.append("%s%s*" % (data_dir, os.path.sep))
+for data_dir in ["magic", "config"]:
+    install_data_files.append("%s%s*" % (data_dir, "/"))
 
 # If doing a build or installation, then create a version.py file
 # which defines the current binwalk version. This file is excluded
 # from git in the .gitignore file.
-if 'install' in ' '.join(sys.argv) or 'build' in ' '.join(sys.argv):
-    sys.stdout.write("creating %s\n" % (VERSION_FILE))
+try:
+    with open(VERSION_FILE, "w") as fp:
+        fp.write("# This file has been auto-generated by setup.py\n")
+        fp.write('__version__ = "%s"\n' % MODULE_VERSION)
+except IOError as e:
+    sys.stderr.write("failed to create file %s: %s\n" % (VERSION_FILE, str(e)))
+    sys.exit(1)
+
+
+scriptPath=os.path.join("src", "scripts", SCRIPT_NAME)
+url="https://github.com/ReFirmLabs/%s" % MODULE_NAME
+
+
+cfg["version"] = MODULE_VERSION
+cfg["package_dir"] = {"":PACKAGE_DIR}
+cfg["packages"] = [MODULE_NAME, MODULE_NAME+".core", MODULE_NAME+".modules", MODULE_NAME+".plugins"]
+#cfg["package_data"]={MODULE_NAME: install_data_files},
+cfg["cmdclass"] = {
+    'clean': CleanCommand,
+    'uninstall': UninstallCommand,
+    'idainstall': IDAInstallCommand,
+    'idauninstall': IDAUnInstallCommand,
+    'autocomplete' : AutoCompleteCommand,
+    'test': TestCommand
+}
 
-    try:
-        with open(VERSION_FILE, "w") as fp:
-            fp.write("# This file has been auto-generated by setup.py\n")
-            fp.write('__version__ = "%s"\n' % MODULE_VERSION)
-    except IOError as e:
-        sys.stderr.write("failed to create file %s: %s\n" % (VERSION_FILE, str(e)))
-        sys.exit(1)
-
-# Install the module, script, and support files
-setup(
-    name=MODULE_NAME,
-    version=MODULE_VERSION,
-    description="Firmware analysis tool",
-    author="Craig Heffner",
-    url="https://github.com/ReFirmLabs/%s" % MODULE_NAME,
-    requires=[],
-    package_dir={"": "src"},
-    packages=[MODULE_NAME],
-    package_data={MODULE_NAME: install_data_files},
-    scripts=[
-        os.path.join(
-            "src",
-            "scripts",
-            SCRIPT_NAME)],
-    cmdclass={
-        'clean': CleanCommand,
-        'uninstall': UninstallCommand,
-        'idainstall': IDAInstallCommand,
-        'idauninstall': IDAUnInstallCommand,
-        'autocomplete' : AutoCompleteCommand,
-        'test': TestCommand})
+setup(**cfg)
diff --git a/src/scripts/binwalk b/src/binwalk/__main__.py
old mode 100755
new mode 100644
similarity index 100%
rename from src/scripts/binwalk
rename to src/binwalk/__main__.py
-- 
2.17.1.windows.2

[KOLANICH]

@devttys0

[CoffeeExpress]

@KOLANICH Thanks for the patch. We'll take a look and see what we can do to get this merged.

[Xyc0]

I got halfway into patching this myself before thinking to check for PRs.
It's a good fix, please consider merging.

[E3V3A]

@CoffeeExpress @devttys0
Could someone please apply the patch?

EDIT: Don't... there's something FUBAR here

[E3V3A]

@KOLANICH
I'm trying to fix this patch, and managed to get it installed, but now I get:

$ binwalk
Traceback (most recent call last):  
File "/usr/bin/binwalk", line 7, in
from binwalk.__init__ import main
    ImportError: cannot import name 'main'

Something is not found or named properly.
What's missing here?

cat /usr/bin/binwalk                                                                                                                                 #!/usr/bin/python3
# -*- coding: utf-8 -*-
import re
import sys 
from binwalk.__init__ import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw?\|\.exe)?$', '', sys.argv[0])
sys.exit(main())

[KOLANICH]

@E3V3A

from binwalk.__init__ import main

should be

from binwalk.__main__ import main

according to

[options.entry_points]
console_scripts = binwalk = binwalk.__main__:main

in setup.cfg

[E3V3A]

@KOLANICH
@devttys0
@CoffeeExpress

Yeah, I noticed...while you were writing. :)
But there were a bunch of other funny things going on, that would not play well.

Added email, URL and platform and requirements etc. so that pip show binwalk is correct.

But now its all fixed, and you can just do:

git clone ...
cd binwalk
pip install .

and you can run binwalk!
and you can remove binwalk with:

pip uninstall binwalk

and it will remove everything correctly without leftover artifacts.
🎉