diff --git a/.travis-script-2.sh b/.travis-script-2.sh index 15aca56..c5b43ce 100755 --- a/.travis-script-2.sh +++ b/.travis-script-2.sh @@ -5,5 +5,5 @@ set -x -e # Show commands being executed and exit nonzero upon errors. ./python2.7/mt.py --help for F in python2.7/*; do - pep8 --ignore=E402,E501 $F + flake8 --ignore=E402,E501 $F done diff --git a/.travis-script-3.sh b/.travis-script-3.sh index d933f51..1abf542 100755 --- a/.travis-script-3.sh +++ b/.travis-script-3.sh @@ -13,5 +13,5 @@ set -x -e # Show commands being executed and exit nonzero upon errors. ./python3/merge-mutt-contacts.py --help for F in generate-readme.py python3/*; do - pep8 --ignore=E402,E501 $F + flake8 --ignore=E402,E501 $F done diff --git a/.travis.yml b/.travis.yml index 3f7ec4c..17a7386 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,18 @@ +sudo: False language: python python: - "2.7" - "3.4" +addons: + apt: + packages: + - gfortran + - libatlas-base-dev install: - - sudo apt-get -qq install libatlas-base-dev gfortran - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-2.txt; fi - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then travis_wait pip install -r requirements-3.txt; fi - - pip install pep8 + - pip install flake8 # Ensure `requirements.txt` contains all of the dependencies # for the scripts and that scripts that only operate on diff --git a/LICENSE.mit b/LICENSE.mit index aba7989..15f69c3 100644 --- a/LICENSE.mit +++ b/LICENSE.mit @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013 Brandon Amos +Copyright (c) 2013-2016 Brandon Amos Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 891afc4..f8893fb 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,22 @@ and inserted directly into the README as markdown. +## [python2.7/music-organizer.py](https://github.com/bamos/python-scripts/blob/master/python2.7/music-organizer.py) ++ Authors: [Brandon Amos](http://bamos.github.io) ++ Created: 2014.04.19 + + +This script (music-organizer.py) organizes my music collection for +iTunes and [mpv](http://mpv.io) using tag information. +The directory structure is `/`, where `` and `` +are lower case strings separated by dashes. + +See my blog post +[Using Python to organize a music directory](http://bamos.github.io/2014/07/05/music-organizer/) +for a more detailed overview of this script. + + + ## [python2.7/mt.py](https://github.com/bamos/python-scripts/blob/master/python2.7/mt.py) + Authors: [Brandon Amos](http://bamos.github.io) + Created: 2014.11.30 @@ -77,39 +93,96 @@ of the output. -## [python2.7/music-organizer.py](https://github.com/bamos/python-scripts/blob/master/python2.7/music-organizer.py) +## [python2.7/caffe-compute-image-mean.py](https://github.com/bamos/python-scripts/blob/master/python2.7/caffe-compute-image-mean.py) + Authors: [Brandon Amos](http://bamos.github.io) -+ Created: 2014.04.19 ++ Created: 2015.08.10 -This script (music-organizer.py) organizes my music collection for -iTunes and [mpv](http://mpv.io) using tag information. -The directory structure is `/`, where `` and `` -are lower case strings separated by dashes. +This script computes the mean of a directory of images for Caffe. -See my blog post -[Using Python to organize a music directory](http://bamos.github.io/2014/07/05/music-organizer/) -for a more detailed overview of this script. +## [python2.7/fix-music-tags.py](https://github.com/bamos/python-scripts/blob/master/python2.7/fix-music-tags.py) ++ Authors: [Brandon Amos](http://bamos.github.io) ++ Created: 2015.12.30 -## [python3/eval-expr.py](https://github.com/bamos/python-scripts/blob/master/python3/eval-expr.py) -+ Authors: J. Sebastian, [Brandon Amos](http://bamos.github.io) -+ Created: 2013.08.01 +This script (fix-music-tags.py) mass-removes unwanted music tags. -A module to evaluate a mathematical expression using Python's AST. -+ Original by: J. Sebastian at http://stackoverflow.com/questions/2371436. -+ Modifications by: [Brandon Amos](http://bamos.github.io). -If you want a command-line expression evaluator, use -[Russell91/pythonpy](https://github.com/Russell91/pythonpy). +## [python3/github-repo-summary.py](https://github.com/bamos/python-scripts/blob/master/python3/github-repo-summary.py) ++ Authors: [Brandon Amos](http://bamos.github.io) ++ Created: 2014.11.02 + + +Produces a Markdown table concisely summarizing a list of GitHub repositories. + + + +## [python3/link-checker.py](https://github.com/bamos/python-scripts/blob/master/python3/link-checker.py) ++ Authors: [Brandon Amos](http://bamos.github.io) ++ Created: 2014.02.06 + + +Script to be run by crontab to report broken links. + +Builds upon linkchecker (Ubuntu: sudo apt-get install linkchecker) +to hide warnings and to send a concise email if bad links are found. + +![Link checker screenshot](https://raw.githubusercontent.com/bamos/python-scripts/master/link-checker-screenshot.png?raw=true) + +## [python3/phonetic.py](https://github.com/bamos/python-scripts/blob/master/python3/phonetic.py) ++ Authors: [Brandon Amos](http://bamos.github.io) ++ Created: 2014.02.14 + + +Obtain the NATO phonetic alphabet representation from short phrases. + ``` -$ eval-expr.py '(((4+6)*10)<<2)' -(((4+6)*10)<<2) = 400 +$ phonetic.py github +g - golf +i - india +t - tango +h - hotel +u - uniform +b - bravo +``` + + + +## [python3/rank-writing.py](https://github.com/bamos/python-scripts/blob/master/python3/rank-writing.py) ++ Authors: [Brandon Amos](http://bamos.github.io) ++ Created: 2014.02.14 + + +`rank-writing.py` ranks the writing quality of my +blog's Markdown posts and my project's Markdown README files. + +The following programs should be on your `PATH`: ++ [aspell](http://aspell.net/) ++ [write-good](https://github.com/btford/write-good) ++ [diction](https://www.gnu.org/software/diction/) + + +``` +$ rank-writing.py *.md + +=== 2013-05-03-scraping-tables-python.md === +Total: 53 +├── aspell: 34 +├── diction: 0 +└── write-good: 19 + +... + +=== 2013-04-16-pdf-from-plaintext.md === +Total: 0 +├── aspell: 0 +├── diction: 0 +└── write-good: 0 ``` @@ -156,7 +229,7 @@ $ rm $(get-osx-wallpaper.py) && killall Dock ``` Example alias definitions for bash and zsh are available in -https://github.com/bamos/dotfiles/blob/master/.aliases: +https://github.com/bamos/dotfiles/blob/master/.funcs: ``` alias open-wallpaper='open $(get-osx-wallpaper.py)' @@ -165,38 +238,6 @@ alias rm-wallpaper='rm $(get-osx-wallpaper.py) && killall Dock' -## [python3/github-repo-summary.py](https://github.com/bamos/python-scripts/blob/master/python3/github-repo-summary.py) -+ Authors: [Brandon Amos](http://bamos.github.io) -+ Created: 2014.11.02 - - -Produces a Markdown table concisely summarizing a list of GitHub repositories. - - - -## [python3/link-checker.py](https://github.com/bamos/python-scripts/blob/master/python3/link-checker.py) -+ Authors: [Brandon Amos](http://bamos.github.io) -+ Created: 2014.02.06 - - -Script to be run by crontab to report broken links. - -Builds upon linkchecker (Ubuntu: sudo apt-get install linkchecker) -to hide warnings and to send a concise email if bad links are found. - -![Link checker screenshot](https://raw.githubusercontent.com/bamos/python-scripts/master/link-checker-screenshot.png?raw=true) - - - -## [python3/merge-mutt-contacts.py](https://github.com/bamos/python-scripts/blob/master/python3/merge-mutt-contacts.py) -+ Authors: [Brandon Amos](http://bamos.github.io) -+ Created: 2014.01.08 - - -Merges two mutt contact files. - - - ## [python3/merge-pdfs-printable.py](https://github.com/bamos/python-scripts/blob/master/python3/merge-pdfs-printable.py) + Authors: [Brandon Amos](http://bamos.github.io) + Created: 2014.10.17 @@ -234,59 +275,6 @@ PS file. -## [python3/phonetic.py](https://github.com/bamos/python-scripts/blob/master/python3/phonetic.py) -+ Authors: [Brandon Amos](http://bamos.github.io) -+ Created: 2014.02.14 - - -Obtain the NATO phonetic alphabet representation from short phrases. - -``` -$ phonetic.py github -g - golf -i - india -t - tango -h - hotel -u - uniform -b - bravo -``` - - - -## [python3/rank-writing.py](https://github.com/bamos/python-scripts/blob/master/python3/rank-writing.py) -+ Authors: [Brandon Amos](http://bamos.github.io) -+ Created: 2014.02.14 - - -`rank-writing.py` ranks the writing quality of my -blog's Markdown posts and my project's Markdown README files. - -The following programs should be on your `PATH`: -+ [aspell](http://aspell.net/) -+ [write-good](https://github.com/btford/write-good) -+ [diction](https://www.gnu.org/software/diction/) - - -``` -$ rank-writing.py *.md - -=== 2013-05-03-scraping-tables-python.md === -Total: 53 -├── aspell: 34 -├── diction: 0 -└── write-good: 19 - -... - -=== 2013-04-16-pdf-from-plaintext.md === -Total: 0 -├── aspell: 0 -├── diction: 0 -└── write-good: 0 -``` - - - ## [python3/remove-duplicates.py](https://github.com/bamos/python-scripts/blob/master/python3/remove-duplicates.py) + Authors: [Brandon Amos](http://bamos.github.io) + Created: 2015.06.06 @@ -329,6 +317,36 @@ $ word-counter.py shakespeare.md --numWords 4 --maxTuples 3 +## [python3/eval-expr.py](https://github.com/bamos/python-scripts/blob/master/python3/eval-expr.py) ++ Authors: J. Sebastian, [Brandon Amos](http://bamos.github.io) ++ Created: 2013.08.01 + + +A module to evaluate a mathematical expression using Python's AST. + ++ Original by: J. Sebastian at http://stackoverflow.com/questions/2371436. ++ Modifications by: [Brandon Amos](http://bamos.github.io). + +If you want a command-line expression evaluator, use +[Russell91/pythonpy](https://github.com/Russell91/pythonpy). + + +``` +$ eval-expr.py '(((4+6)*10)<<2)' +(((4+6)*10)<<2) = 400 +``` + + + +## [python3/merge-mutt-contacts.py](https://github.com/bamos/python-scripts/blob/master/python3/merge-mutt-contacts.py) ++ Authors: [Brandon Amos](http://bamos.github.io) ++ Created: 2014.01.08 + + +Merges two mutt contact files. + + + # Similar Projects There are many potpourri Python script repositories on GitHub. The following list shows a short sampling of projects, @@ -336,8 +354,8 @@ and I'm happy to merge pull requests of other projects. Name | Stargazers | Description ----|----|---- -[averagesecurityguy/Python-Examples](https://github.com/averagesecurityguy/Python-Examples) | 20 | Example scripts for common python tasks -[ClarkGoble/Scripts](https://github.com/ClarkGoble/Scripts) | 26 | My scripts - primarily using python and appscript -[computermacgyver/twitter-python](https://github.com/computermacgyver/twitter-python) | 45 | Simple example scripts for Twitter data collection with Tweepy in Python -[gpambrozio/PythonScripts](https://github.com/gpambrozio/PythonScripts) | 39 | A bunch of Python scripts I made and that might interest somebody else -[realpython/python-scripts](https://github.com/realpython/python-scripts) | 59 | because i'm tired of gists +[averagesecurityguy/Python-Examples](https://github.com/averagesecurityguy/Python-Examples) | 26 | Example scripts for common python tasks +[ClarkGoble/Scripts](https://github.com/ClarkGoble/Scripts) | 29 | My scripts - primarily using python and appscript +[computermacgyver/twitter-python](https://github.com/computermacgyver/twitter-python) | 66 | Simple example scripts for Twitter data collection with Tweepy in Python +[gpambrozio/PythonScripts](https://github.com/gpambrozio/PythonScripts) | 38 | A bunch of Python scripts I made and that might interest somebody else +[realpython/python-scripts](https://github.com/realpython/python-scripts) | 568 | because i'm tired of gists diff --git a/python2.7/caffe-compute-image-mean.py b/python2.7/caffe-compute-image-mean.py new file mode 100755 index 0000000..d5b8540 --- /dev/null +++ b/python2.7/caffe-compute-image-mean.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python2 + +__author__ = ['[Brandon Amos](http://bamos.github.io)'] +__date__ = '2015.08.10' + +""" +This script computes the mean of a directory of images for Caffe. +""" + +import sys +sys.path.append("/home/bamos/repos/caffe-local/python") + +import argparse +import numpy as np +import os +import time + +from caffe.io import array_to_blobproto +from collections import defaultdict +from skimage import io + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('meanPrefix', type=str, help="Prefix of the mean file.") + parser.add_argument('imageDir', type=str, help="Directory of images to read.") + args = parser.parse_args() + + exts = ["jpg", "png"] + + mean = np.zeros((1, 3, 152, 152)) + N = 0 + classSizes = defaultdict(int) + + beginTime = time.time() + for subdir, dirs, files in os.walk(args.imageDir): + for fName in files: + (imageClass, imageName) = (os.path.basename(subdir), fName) + if any(imageName.lower().endswith("." + ext) for ext in exts): + img = io.imread(os.path.join(subdir, fName)) + if img.shape == (152, 152, 3): + mean[0][0] += img[:, :, 0] + mean[0][1] += img[:, :, 1] + mean[0][2] += img[:, :, 2] + N += 1 + if N % 1000 == 0: + elapsed = time.time() - beginTime + print("Processed {} images in {:.2f} seconds. " + "{:.2f} images/second.".format(N, elapsed, + N / elapsed)) + mean[0] /= N + + blob = array_to_blobproto(mean) + with open("{}.binaryproto".format(args.meanPrefix), 'wb') as f: + f.write(blob.SerializeToString()) + np.save("{}.npy".format(args.meanPrefix), mean[0]) + + meanImg = np.transpose(mean[0].astype(np.uint8), (1, 2, 0)) + io.imsave("{}.png".format(args.meanPrefix), meanImg) diff --git a/python2.7/fix-music-tags.py b/python2.7/fix-music-tags.py new file mode 100755 index 0000000..8e07eb2 --- /dev/null +++ b/python2.7/fix-music-tags.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python2 + +__author__ = ['[Brandon Amos](http://bamos.github.io)'] +__date__ = '2015.12.30' + +""" +This script (fix-music-tags.py) mass-removes unwanted music tags. +""" + +from mutagen.easyid3 import EasyID3 +import argparse +import glob + + +def fixTags(fname, keep): + audio = EasyID3(fname) + + delKeys = [] + for k, v in audio.items(): + if k not in keep: + delKeys.append(k) + + for k in delKeys: + del audio[k] + audio.save() + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('directory', help='Directory with mp3 files to fix.') + parser.add_argument('--keep', default=['title', 'artist', 'album', 'genre'], + type=str, nargs='+', metavar='TAG', + help="Tags to keep. Default: title, artist, album, genre") + args = parser.parse_args() + + for fname in glob.glob("{}/*.mp3".format(args.directory)): + print("Fixing tags for {}".format(fname)) + fixTags(fname, args.keep) diff --git a/python2.7/music-autoplaylists.py b/python2.7/music-autoplaylists.py new file mode 100755 index 0000000..15759bf --- /dev/null +++ b/python2.7/music-autoplaylists.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python2.7 + +__author__ = ['[Brandon Amos](http://bamos.github.io)'] +__date__ = '2015.04.09' + +""" +This script (music-autoplaylists.py) automatically creates +M3U playlists from the genre ID3 tags of songs in a directory. +""" + +import argparse +import os +import re +import shutil +import sys +from mutagen.easyid3 import EasyID3 +from collections import defaultdict + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--musicDir', type=str, default='.') + parser.add_argument('--playlistDir', type=str, default='./playlists/auto') + args = parser.parse_args() + + genres = defaultdict(list) + for dpath, dnames, fnames in os.walk(args.musicDir): + if '.git' in dpath: + continue + for fname in fnames: + if os.path.splitext(fname)[1] != '.mp3': + continue + p = os.path.abspath(os.path.join(dpath, fname)) + audio = EasyID3(p) + if 'genre' in audio: + assert(len(audio['genre']) == 1) + genre = toNeat(str(audio['genre'][0])) + else: + genre = 'Unknown' + genres[genre].append(p) + + if os.path.exists(args.playlistDir): + shutil.rmtree(args.playlistDir) + os.makedirs(args.playlistDir) + + for genre, songs in genres.items(): + p = os.path.join(args.playlistDir, genre + '.m3u') + print("Creating playlist: {}".format(p)) + with open(p, 'w') as f: + f.write("#EXTM3U\n") + f.write("\n".join(sorted(songs)) + "\n") + +# Maps a string such as 'The Beatles' to 'the-beatles'. + + +def toNeat(s): + s = s.lower().replace("&", "and") + + # Put spaces between and remove blank characters. + blankCharsPad = r"()\[\],.\\\?\#/\!\$\:\;" + blankCharsNoPad = r"'\"" + s = re.sub(r"([" + blankCharsPad + r"])([^ ])", "\\1 \\2", s) + s = re.sub("[" + blankCharsPad + blankCharsNoPad + "]", "", s) + + # Replace spaces with a single dash. + s = re.sub(r"[ \*\_]+", "-", s) + s = re.sub("-+", "-", s) + s = re.sub("^-*", "", s) + s = re.sub("-*$", "", s) + + # Ensure the string is only alphanumeric with '-', '+', and '='. + search = re.search("[^0-9a-z\-\+\=]", s) + if search: + print("Error: Unrecognized character in '" + s + "'") + sys.exit(-42) + return s + +if __name__ == '__main__': + main() diff --git a/python2.7/music-organizer.py b/python2.7/music-organizer.py index 7437f1e..1bed95e 100755 --- a/python2.7/music-organizer.py +++ b/python2.7/music-organizer.py @@ -15,6 +15,7 @@ """ import argparse +import glob import os import re import shutil @@ -65,7 +66,7 @@ def toNeat(s): s = s.lower().replace("&", "and") # Put spaces between and remove blank characters. - blankCharsPad = r"()\[\],.\\\?\#/\!\$\:" + blankCharsPad = r"()\[\],.\\\?\#/\!\$\:\;" blankCharsNoPad = r"'\"" s = re.sub(r"([" + blankCharsPad + r"])([^ ])", "\\1 \\2", s) s = re.sub("[" + blankCharsPad + blankCharsNoPad + "]", "", s) @@ -194,7 +195,7 @@ def song(filename): def collection(): - for f in os.listdir("."): + for f in glob.glob('*'): if os.path.isdir(f): if f != 'iTunes' and f != 'playlists': artist(f) diff --git a/python3/get-osx-wallpaper.py b/python3/get-osx-wallpaper.py index 74e349c..1feb0dc 100755 --- a/python3/get-osx-wallpaper.py +++ b/python3/get-osx-wallpaper.py @@ -41,7 +41,7 @@ ``` Example alias definitions for bash and zsh are available in -https://github.com/bamos/dotfiles/blob/master/.aliases: +https://github.com/bamos/dotfiles/blob/master/.funcs: ``` alias open-wallpaper='open $(get-osx-wallpaper.py)' diff --git a/python3/github-repo-summary.py b/python3/github-repo-summary.py index aece884..e59e96c 100755 --- a/python3/github-repo-summary.py +++ b/python3/github-repo-summary.py @@ -9,7 +9,6 @@ from github import Github import argparse -import time import os import sys @@ -30,8 +29,7 @@ def sanitize_for_md(s): try: r = github.get_repo(r_name) except: - print("Error: Repository '{}' not found.".format(r_name), - file=sys.stderr) + sys.stderr.write("Error: Repository '{}' not found.\n".format(r_name)) sys.exit(-1) content = " | ".join([ "[{}]({})".format(r.full_name, r.html_url), diff --git a/python3/link-checker.py b/python3/link-checker.py index 097d74f..846ff66 100755 --- a/python3/link-checker.py +++ b/python3/link-checker.py @@ -21,13 +21,13 @@ # Settings to send emails with SMTP with gmail. server = "smtp.gmail.com" port = 587 -user = ENTER_USER -pw = ENTER_PW # Please use an application-specific password for security! -email_to = ENTER_TO_EMAIL -email_from = ENTER_FROM_EMAIL -root_url = ENTER_URL +user = 'ENTER_USER' +pw = 'ENTER_PW' # Please use an application-specific password for security! +email_to = 'ENTER_TO_EMAIL' +email_from = 'ENTER_FROM_EMAIL' +root_url = 'ENTER_URL' -cmd = ["linkchecker", "--no-warnings", "--no-status", root_url] +cmd = ["linkchecker", "--no-warnings", "--no-status", "--external", root_url] output = Popen(cmd, stdout=PIPE).communicate()[0].decode("UTF-8") bad_urls = [] diff --git a/python3/merge-pdfs-printable.py b/python3/merge-pdfs-printable.py index c88a840..d2c2cd4 100755 --- a/python3/merge-pdfs-printable.py +++ b/python3/merge-pdfs-printable.py @@ -37,7 +37,6 @@ import argparse import os -import re import subprocess import tempfile @@ -87,7 +86,7 @@ def merge_pdfs(f_names): out_file = "merged.pdf" # tempfile.mktemp("-merge.pdf") with open(out_file, 'wb') as f: merger.write(f) - [f.close() for f in fps] + [fp.close() for fp in fps] print("Merged output is in '{}'.".format(out_file)) if __name__ == '__main__': diff --git a/python3/remove-duplicates.py b/python3/remove-duplicates.py index 1158f2f..d2bb442 100755 --- a/python3/remove-duplicates.py +++ b/python3/remove-duplicates.py @@ -13,7 +13,6 @@ import hashlib import imagehash import os -import sys from collections import defaultdict from PIL import Image @@ -27,7 +26,8 @@ def getImgs(d): for subdir, dirs, files in os.walk(d): imgs = [] for fName in files: - (imageClass, imageName) = (os.path.basename(subdir), fName) + # (imageClass, imageName) = (os.path.basename(subdir), fName) + imageName = fName if any(imageName.lower().endswith("." + ext) for ext in exts): imgs.append(os.path.join(subdir, fName)) imgClasses.append(imgs)