Create a Websocket endpoint in Python with Postgres.
Real-time Notifications with Python and Postgres
DB
Running Standalone SQLAlchemy Scripts in Pyramid
From time to time there comes the need to run automated scripts managed by either Python or Cron. In a recent project I had to run a standalone script that checks records on a database at certain hours. So here’s what I came up with. Most of the script is based on this post by Daniel Mayer.
from paste.deploy import appconfig
from sqlalchemy import engine_from_config
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
#here is where sqlalchemy objects are imported
from PRJ.models import DBSession as db,DBRecords
#import the session manager. This way commits will be handled automatically by the Zope Transaction Manager
import transaction
# Load Application Configuration and Return as Dictionary
conf = appconfig('config:' + 'development.ini',
relative_to="/var/www/PRJ",
name="main")
# Bind Engine Based on Config
engine = engine_from_config(conf, 'sqlalchemy.')
db.configure(bind=engine)
#Query the DB
data = db.query(DBRecords).one()
with transaction.manager:
for record in data:
#query or update DB
Host Email with Postfix, Dovecot and Postgres on Debian 6 (Squeeze)
Recently I had to configure a web server from scratch, which I haven’t done in a while. So I took a look at the guides from my current hosting provider Linode. They have a guide covering debian 5 (Lenny) which is basically the same for debian 6 (squeeze). To make this a bit more exciting I decided to replace MySQL for Postgres since I really didn’t need it. So I will only post about the changes you need to perform based from the Linode guide.
Install Required Packages
Replace the mysql related packages for postgres and replace mailx for bsd-mailx.
apt-get install postfix postfix-pgsql postfix-doc dovecot-common dovecot-imapd dovecot-pop3d postfix-tls libsasl2-2 libsasl2-modules libsasl2-modules-sql sasl2-bin libpam-pgsql openssl telnet bsd-mailx postgresql-contrib postgresql-8.4
Set up Postgres for Virtual Domains and Users
Open /etc/postgres/8.4/main/pg_hba.conf and make sure local connections require a password
host all all 127.0.0.1/32 md5
You could also make all local connections trusted and in this case no password will be needed, but of course it could represent a security risk
host all all 127.0.0.1/32 trust
This next step is optional but as a security measure I like to have Postgres listen on a diferent port other than the default.
Open /etc/postgres/8.4/main/postgres.conf and uncomment the following line and add 6789 as the port number
port = 6789
Create the user for the mail database.
sudo -u postgres createuser -P -E -s -d mail_admin
Now create the database.
sudo -u postgres createdb -O mail_admin mail -E UTF-8
Grant all privileges.
GRANT ALL PRIVILEGES ON DATABASE mail to mail_admin;
Create domains, forwardings, users and transport tables.
CREATE TABLE domains (domain varchar(50) NOT NULL, PRIMARY KEY (domain) );
CREATE TABLE forwardings (source varchar(80) NOT NULL, destination TEXT NOT NULL, PRIMARY KEY (source) );
CREATE TABLE users (email varchar(80) NOT NULL, password varchar(20) NOT NULL, PRIMARY KEY (email) );
CREATE TABLE transport ( domain varchar(128) NOT NULL default '', transport varchar(128) NOT NULL default '', constraint fieldomainunique UNIQUE (domain) );
Now I’ll create a domain and user in mail database. But first I will use Postgres pgcrypto functions since passwords need to be encrypted using the UNIX original encryption scheme. Replace your_password with your own password.
mail=#\i /usr/share/postgresql/8.4/contrib/pgcrypto.sql
INSERT INTO domains (domain) VALUES ('example.com');
INSERT INTO users (email, password) VALUES ('[email protected]', crypt('your_password', gen_salt('des') ));
mail=#\q
Configure Postfix to work with Postgres
Create file /etc/postfix/pgsql-virtual_domains.cf
user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT domain AS virtual FROM domains WHERE domain='%s'
hosts = 127.0.0.1:6789
Create file /etc/postfix/pgsql-virtual_forwardings.cf
user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT destination FROM forwardings WHERE source='%s'
hosts = 127.0.0.1:6789
Create file /etc/postfix/pgsql-virtual_mailboxes.cf
user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT split_part(email, '@', 2) || '/' || split_part(email,'@', 1) FROM users WHERE email='%s';
hosts = 127.0.0.1:6789
Create file /etc/postfix/pgsql-virtual_email2email.cf
user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT email FROM users WHERE email='%s'
hosts = 127.0.0.1:6789
Set the proper permissions
chmod o= /etc/postfix/pgsql-virtual_*.cf
chgrp postfix /etc/postfix/pgsql-virtual_*.cf
Add the following one line at a time. Replace qualified.hostname.com with your hostname
postconf -e 'myhostname = qualified.hostname.com'
postconf -e 'mydestination = qualified.hostname.com, localhost, localhost.localdomain'
postconf -e 'mynetworks = 127.0.0.0/8'
postconf -e 'message_size_limit = 30720000'
postconf -e 'virtual_alias_domains ='
postconf -e 'virtual_alias_maps = proxy:pgsql:/etc/postfix/pgsql-virtual_forwardings.cf, pgsql:/etc/postfix/pgsql-virtual_email2email.cf'
postconf -e 'virtual_mailbox_domains = proxy:pgsql:/etc/postfix/pgsql-virtual_domains.cf'
postconf -e 'virtual_mailbox_maps = proxy:pgsql:/etc/postfix/pgsql-virtual_mailboxes.cf'
postconf -e 'virtual_mailbox_base = /home/vmail'postconf -e 'virtual_uid_maps = static:5000'postconf -e 'virtual_gid_maps = static:5000'
postconf -e 'smtpd_sasl_auth_enable = yes'postconf -e 'broken_sasl_auth_clients = yes'
postconf -e 'smtpd_sasl_authenticated_header = yes'
postconf -e 'smtpd_recipient_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_destination'
postconf -e 'smtpd_use_tls = yes'
postconf -e 'smtpd_tls_cert_file = /etc/postfix/smtpd.cert'
postconf -e 'smtpd_tls_key_file = /etc/postfix/smtpd.key'
postconf -e 'virtual_create_maildirsize = yes'
postconf -e 'virtual_maildir_extended = yes'
postconf -e 'proxy_read_maps = $local_recipient_maps $mydestination $virtual_alias_maps $virtual_alias_domains $virtual_mailbox_maps $virtual_mailbox_domains $relay_recipient_maps $relay_domains $canonical_maps $sender_canonical_maps $recipient_canonical_maps $relocated_maps $transport_maps $mynetworks $virtual_mailbox_limit_maps'
postconf -e virtual_transport=dovecotpostconf -e dovecot_destination_recipient_limit=1
Change postfix listening port to 2525 in /etc/postfix/master.cf.
2525 inet n - - - - smtpd
Or you could just uncomment the submission line and default to port 587.
submission inet n - - - - smtpd
Configure saslauthd to use Postgres
Create file /etc/pam.d/smtp. Replace your_password with your own password.
auth required pam_pgsql.so user=mail_admin passwd=your_password host=127.0.0.1 port=6789 db=mail table=users usercolumn=email passwdcolumn=password crypt=1
account sufficient pam_pgsql.so user=mail_admin passwd=your_password host=127.0.0.1 port=6789 db=mail table=users usercolumn=email passwdcolumn=password crypt=1
Create file /etc/postfix/sasl/smtpd.conf. Replace your_password with your own password.
pwcheck_method: saslauthd
mech_list: plain login
allow_plaintext: true
auxprop_plugin: sql
sql_engine: pgsql
sql_hostnames: 127.0.0.1:6789
sql_user: mail_admin
sql_passwd: your_password
sql_database: mail
sql_select: select password from users where email = '%u'
Create file /etc/pam_pgsql.conf. Replace your_password with your own password.
database = mail
host = localhost
user = mail_admin
password = your_password
table = users
user_column = email
pwd_column = password
pw_type = crypt
Set proper permissions to files.
chmod o= /etc/pam_pgsql.conf
chmod o= /etc/postfix/sasl/smtpd.conf
Configure Dovecot
One thing that was pointed out at the end of the guide by user Nathan is to change cmusieve for sieve on file /etc/dovecot/dovecot.conf on this following part
protocol lda {
log_path = /home/vmail/dovecot-deliver.log
auth_socket_path = /var/run/dovecot/auth-master
postmaster_address = [email protected]
mail_plugins = cmusieve #change to sieve
global_script_path = /home/vmail/globalsieverc
}
Create file /etc/dovecot/dovecot-sql.conf. Replace your_password with your own password.
driver = pgsql
connect = host=127.0.0.1 port=6789 dbname=mail user=mail_admin password=your_password
default_pass_scheme = CRYPT
password_query = SELECT email as user, password FROM users WHERE email='%u';
And thats about it. If you managed to configure everything correctly congratulations!, if not post a comment and we’ll try to find the solution.
Troubleshooting
Remember that /var/log/syslog and /var/log/mail.log are your friends. You can also add the word debug at the end of each line on /etc/pam.d/smtp to help you troubleshoot any problems you may encounter.
repoze.what with reflected tables using SQLAlchemy in Pylons
One of the many things I like about SQLAlchemy is the feature of reflected tables, this means I can develop dashboards for existing applications with my current favorite framework(Pylons). So I had to create an authorization mechanism for my dashboard and this recipe from the Pylons cookbook was just what I needed.
The only thing I had to change was the way my tables were being mapped since the users table schema was already defined. This is what I did:
user_table = schema.Table('users', Base.metadata, autoload=True, autoload_with=engine)
user_group_table = schema.Table('user_groups',Base.metadata,autoload=True,autoload_with=engine)
group_permission_table = schema.Table('group_permission',Base.metadata,autoload=True,autoload_with=engine)
permission_table = schema.Table('permissions', Base.metadata,autoload=True,autoload_with=engine)
group_table = schema.Table('groups', Base.metadata,autoload=True,autoload_with=engine)
orm.mapper(User,user_table, properties={
'groups':orm.relation(Group, secondary=user_group_table),
})
orm.mapper(Permission,permission_table,properties={
'groups':orm.relation(Group, secondary=group_permission_table),
})
orm.mapper(Group,group_table,properties={
'permissions':orm.relation(Permission, secondary=group_permission_table),
'users':orm.relation(User, secondary=user_group_table),
})
orm.mapper(UGT,user_group_table)
orm.mapper(GPT,group_permission_table)
And here’s the schema I had to create, except for the users table:
CREATE TABLE user_groups(
id serial NOT NULL,
user_id integer,
group_id integer,
CONSTRAINT user_groups_pkey PRIMARY KEY (id),
CONSTRAINT gid_fkey FOREIGN KEY (group_id) REFERENCES groups (id)
CONSTRAINT uid_fkey FOREIGN KEY (user_id) REFERENCES users (id)
)
CREATE TABLE group_permission(
id serial NOT NULL,
gid integer,
permid integer,
CONSTRAINT group_permission_pkey PRIMARY KEY (id),
CONSTRAINT groupid_fkey FOREIGN KEY (gid) REFERENCES groups (id),
CONSTRAINT permid_fkey FOREIGN KEY (permid) REFERENCES permissions (id)
)
CREATE TABLE permissions(
id serial NOT NULL,
pname character varying(50),
CONSTRAINT permissions_pkey PRIMARY KEY (id)
)
CREATE TABLE groups(
id serial NOT NULL,
gname character varying(50),
CONSTRAINT groups_pkey PRIMARY KEY (id)
)
CREATE TABLE users(
id serial NOT NULL,
username character varying(50),
email character varying(50),
upassword character varying(150),
CONSTRAINT users_pkey PRIMARY KEY (id)
)
And that’s all there’s to it. I was very surprise on how flexible SQLAlchemy is.