Dicts and lists

Iterate over keys and values in a dict

obj = { "key" : val }

# Python 2
for key, val in obj.iteritems():
    print key, val

# Python 3
for key, val in obj.items():
    print(key, val)

Extend / merge a dict with more values

a = { "foo" : 1 }
a.update({ "bar" : 2 })
print(a) # { "foo" : 1, "bar" : 2 }

Get the first item in a dict

a = { "a" : {}, "b" : {} }
a.itervalues().next()

Get the first N items in a dict

{k:v for k,v in list(d.items())[0:N]}

Remove a key from a dict

my_dict.pop("key", None)

Get a default value for a dict if the key does not exist

a = { "foo" : "foo" }
a["foo"] // "foo"
a.get("bar", "bar") // "bar"
a["bar"] // KeyError: "bar"

Get the index number when looping over a list (array)

l = [1,2,3]
for index, val in enumerate(l):
    print(index, val) # "0, 1", "1, 2", "2, 3"

Sort a list by a user defined function

a = [ { "foo" : "bar" }, {"foo" : "baz"} ]
a.sort(key = lambda i:i["foo"])

Or

a = sorted(a, key = lambda i:i["foo"])

Sort a list with dicts by a key in the dicts?

a = [ { "name" : "Bert" }, { "name" : "Ernie" }

from operator import itemgetter
b = sorted(a, key = itemgetter("name") )

Sort a list with lists on one of the items?

a = [ ["foo", 3], ["foo", 1], ["foo", 2] ]
a.sort(key = lambda i:i[1]) // [ ["foo", 1], ["foo", 2], ["foo", 3] ]

Sort a dict by its values?

x = { "a" : 3, "b" : 2, "c" : 9 }
dict(sorted(x.items(), key=lambda item: item[1]))

Get the key of a dict where the value is the largest

a = { "foo" : 10, "bar" : 20, "baz" : 5 }
max(a, key = (lambda k:a[k])) # 'bar'

Count the number of times an item appears in a list?

from collections import Counter

l = ["foo", "bar", "foo", "baz"]
print(Counter(l))

# Prints:
# {'foo': 2, 'baz': 1, 'bar': 1}

Flatten a list with nested lists?

The ugly way

sum(l, [])

The slighty more Pythonic (and faster) way

[item for sublist in l for item in sublist]

And the fastest way

from itertools import chain
list(chain.from_iterable(l))

Get the unique values in a list

Convert to a set, then to a list again

list(set(l))

Remove falsy values from a list

Python 2

values = filter(None, [1, None, 3])

Python 3

values = list(filter(None, [1, None, 3]))

Split a list in evenly sized chunks

lst = list(range(0, 180))
chunks = [ lst[i:i + 50] for i in range(0, len(lst), 50) ]

Or if you want a function:

def batch(iterable, n = 1):
    for i in range(0, len(iterable), n):
        yield iterable[i:i + n]

Filter a list using comprehensions

l = [5, 3, 7, 24, 2, 22]
l = [ x for x in l if x > 10 ] # l == [24, 22]

Dict comprehensions

d = {k:v for k,v in dct.iteritems() } # Python 2
d = {k:v for k,v in dct.items() } # Python 3

Strings

Format a string

Either

"Ham, %s and %s" % ("eggs", "spam")

Or

"Ham, {food1} and {food2}".format(food1 = "eggs", food2 = "spam")

Or (since Python 3.6)

food1 = "eggs"
food2 = "spam"
f"Ham, {food1} and {food2}"

Pad a number with zeroes

If the number is already a string

number.zfill(3)

URL encode a string

urllib.parse.quote_plus("Lots of weird charachters:ª•¶•ª¶™•ª¢∆")

Check if any string in a list appears in another string

target = "a lovely string"
strings = ("nice", "kind", "lovely")

if any(s in target for s in strings):
    print(s) # 'lovely'

Regular expressions

import re
REGEX = re.compile("…[^…]*…")
# Replace all occurences
text = REGEX.sub(" ", text)
# Get all matches
matches = REGEX.findall(text)
# Split by regex
parts = REGEX.split(text)

Dates and times

Get the current time as a ISO8601-formatted string

import datetime
datetime.datetime.now().isoformat("T")

Files

Get the directory of the current executing file

import os
PATH = os.path.dirname(os.path.realpath(__file__))

Or using the more modern way by using pathlib:

from pathlib import Path
PATH = str(Path(__file__).parent)

Check if a file exists?

import os.path
os.path.isfile( filename )

Get a filename without extension

import os
os.path.basename(filename)[0]

To also remove the path to the filename

os.path.splitext(os.path.basename(filename))[0]

Logging

import logging

logger = logging.getLogger(__name__)
logger.critical("Log with critical level")
logger.error("Log with error level")
logger.warning("Log with warning level")
logger.info("Log with info level")
logger.debug("Log with debug level")
logging.basicConfig(level=logging.DEBUG) # Set default level to DEBUG

Classes

class Parent:
    foo = "Unknown"

    def __init__(self, foo):
        self.foo = foo

    def bar(self):
        print(self.foo)

class Child(Parent):
    def __init__(self, foo):
        super().__init__(foo)

Use __getitem__ for easy list access

class Uppercaser:
    def __getitem__(self, key):
        return key.upper()
        
up = Uppercaser()
print(up["hello"]) # HELLO

Use properties (getters and setters)

class Namer:
    def __init__(self, name):
        self._name = name

    @property
    def name(self):
        return self._name.upper()

    @name.setter
    def name(self, value):
        self._name = value

Errors and Exceptions

def test():
    raise Exception("Something happened")

try:
    test()
except Exception as e:
    print(e)
else:
    # Do something else

Note that you can re-raise an exception

try:
    test()
except Exception as e:
    raise e

Custom exceptions

class MyException(Exception):
    pass

try:
    raise MyException("oops")
except MyException as e:
    print("oops")

XML

To parse XML use xmltodict, to make it behave like a dict.

import xmltodict

doc = xmltodict.parse("""
<record id="123">
    <key>foo</key>
</record>
""")

print(doc["record"]["@id"]) # '123'
print(doc["record"]["key"]) # foo

To write back XML you can use xmltodict’s unparse

print(xmltodict.unparse(doc, pretty = True)) # ....

Or use lxml.etree.tostring()

from lxml import etree
print(etree.tostring(doc))

argparse

import argparse

parser = argparse.ArgumentParser(description = "Pretty description here")
parser.add_argument('head', type = str, nargs='?')

parser.add_argument('face', type = str, nargs='?')
parser.add_argument('-o', '--output', type = str, default = "out.jpg")
parser.add_argument('-v', '--verbose', action="store_true")
parser.add_argument('-c', '--caching', choices=('default', 'redis'))

args = parser.parse_args()

if not args.head and args.face:
    parser.print_help()

Virtualenv

Make sure virtualenv is installed

pip install virtualenv

This will create an venv in a directory called env

virtualenv env

Or alternatively

python -m venv env

Then to activate it

source env/bin/activate

Installing local libs (from the venv)

pip install /path/to/lib

Or use venv in python3

python3 -m venv env
source env/bin/activate
pip install -r requirements.txt

Deactivate a venv

deactivate

Using pyenv

Set a local version

pyenv local 3.8.12

List all available installed versions

pyenv versions

List all versions that can be installed

pyenv versions --list

Install a new version

pyenv install 3.8.12

Multithreading

from concurrent.futures import ThreadPoolExecutor

def square(val):
    return val * val

values = [5, 4, 12, 40]

with ThreadPoolExecutor(max_workers = 10) as executor:
    executor.map(square, values)

Other stuff

Simplify a complex loop?

Use a generator. Instead of

for point in points:
    for x in point:
        if x > 5:
            plot(x)

Do

def iterpoints():
    for point in points:
        for x in points:
            if x > 5:
                yield x

for x in iterpoints():
    plot(x)

Increment an integer? foo++ doesn’t work

foo += 1

Or

foo = foo + 1

Break into debugger from the code itself

import pdb
pdb.set_trace()

Or in more recent versions

breakpoint()

Encode URLs?

from urllib import quote_plus as encode
encode('Something with spaces and ••ª¶ª•¶∆˙˚∆ charachters')

Transform a CSV file using dicts

import csv

# Get the fieldnames first
infile = open("in.csv", "r")
reader = csv.DictReader(infile)
fieldnames = reader.next().keys()
infile.close()

# Now open the in and outfile
infile = open("in.csv", "r")
outfile = open("out.csv", "w")
reader = csv.DictReader(infile)
writer = csv.DictWriter(outfile, fieldnames = fieldnames)
# Print the field names in the first row

writer.writerow(
    dict(
        (fn, fn) for fn in fieldnames
    )
)

for row in reader:
    # Do something with the values
    # And then write it
    writer.writerow(row)

infile.close()
outfile.close()

How do i remove the mysterious “ValueError: View function did not return a response” with WSGI?

Check if there’s no print statements in your code.

Parsing HTML using BeautifulSoup

from bs4 import BeautifulSoup

soup = BeautifulSoup(html, 'html5lib')
videos = soup.select(".video")

for v in videos:
    stats = v.select_one(".stats").get_text()

    return {
        "stats" : stats,
        "url" : v.get("href", None),
        "title" : v.select("h1")[0].get_text(),
    }

Download a file over HTTP

import urllib.request
urllib.request.urlretrieve('http://www.example.com/songs/mp3.mp3', 'mp3.mp3')

Execute a shell command

import subprocess
subprocess.check_call(cmd, shell = True)

To do a ‘silent’ call, without any logging

import os, subprocess
subprocess.check_call(cmd, shell = True,
    stdout = open(os.devnull, 'wb')
)

Modules and imports

To import a module from a relative folder (e.g. in test/test.py import from module/command.py), add this to test/test.py:

import sys
sys.path.append("..")