From 526f398695d7385d15b9457c2744e557141c1695 Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Fri, 12 Apr 2019 03:21:20 +1000
Subject: [PATCH 01/12] begin making os independent

---
 mwscrape/scrape.py | 3 ++-
 requirements.txt   | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 requirements.txt

diff --git a/mwscrape/scrape.py b/mwscrape/scrape.py
index ad28591..d7c680f 100644
--- a/mwscrape/scrape.py
+++ b/mwscrape/scrape.py
@@ -5,8 +5,9 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__  import print_function
+from filelock import Timeout, FileLock
+
 import argparse
-import fcntl
 import hashlib
 import os
 import socket
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..666ac7e
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,5 @@
+futures
+CouchDB >= 0.10'
+mwclient >= 0.7.2'
+pylru
+filelock
\ No newline at end of file

From 8b46f2ed171f61920b97c373fbd1fd84107f7745 Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Fri, 12 Apr 2019 03:23:17 +1000
Subject: [PATCH 02/12] tab indentation

---
 mwscrape/scrape.py | 916 ++++++++++++++++++++++-----------------------
 1 file changed, 458 insertions(+), 458 deletions(-)

diff --git a/mwscrape/scrape.py b/mwscrape/scrape.py
index d7c680f..e3e5355 100644
--- a/mwscrape/scrape.py
+++ b/mwscrape/scrape.py
@@ -30,148 +30,148 @@
 
 
 def fix_server_url(general_siteinfo):
-    """
-    Get server url from siteinfo's 'general' dict,
-    add http if scheme is missing. This will also modify
-    given dictionary.
-
-    >>> general_siteinfo = {'server': '//simple.wikipedia.org'}
-    >>> fix_server_url(general_siteinfo)
-    'http://simple.wikipedia.org'
-    >>> general_siteinfo
-    {'server': 'http://simple.wikipedia.org'}
-
-    >>> fix_server_url({'server': 'https://en.wikipedia.org'})
-    'https://en.wikipedia.org'
-
-    >>> fix_server_url({})
-    ''
-
-    """
-    server = general_siteinfo.get('server', '')
-    if server:
-        p = urlparse.urlparse(server)
-        if not p.scheme:
-            server = urlparse.urlunparse(
-                urlparse.ParseResult('http', p.netloc, p.path,
-                                     p.params, p.query, p.fragment))
-            general_siteinfo['server'] = server
-    return server
+	"""
+	Get server url from siteinfo's 'general' dict,
+	add http if scheme is missing. This will also modify
+	given dictionary.
+
+	>>> general_siteinfo = {'server': '//simple.wikipedia.org'}
+	>>> fix_server_url(general_siteinfo)
+	'http://simple.wikipedia.org'
+	>>> general_siteinfo
+	{'server': 'http://simple.wikipedia.org'}
+
+	>>> fix_server_url({'server': 'https://en.wikipedia.org'})
+	'https://en.wikipedia.org'
+
+	>>> fix_server_url({})
+	''
+
+	"""
+	server = general_siteinfo.get('server', '')
+	if server:
+		p = urlparse.urlparse(server)
+		if not p.scheme:
+			server = urlparse.urlunparse(
+				urlparse.ParseResult('http', p.netloc, p.path,
+									 p.params, p.query, p.fragment))
+			general_siteinfo['server'] = server
+	return server
 
 
 def update_siteinfo(site, couch_server, db_name):
-    try:
-        siteinfo_db = couch_server.create('siteinfo')
-    except couchdb.PreconditionFailed:
-        siteinfo_db = couch_server['siteinfo']
+	try:
+		siteinfo_db = couch_server.create('siteinfo')
+	except couchdb.PreconditionFailed:
+		siteinfo_db = couch_server['siteinfo']
 
-    siteinfo = site.api('query', meta='siteinfo',
-                        siprop='general|interwikimap|rightsinfo|statistics|namespaces'
-    )['query']
+	siteinfo = site.api('query', meta='siteinfo',
+						siprop='general|interwikimap|rightsinfo|statistics|namespaces'
+	)['query']
 
-    fix_server_url(siteinfo['general'])
+	fix_server_url(siteinfo['general'])
 
-    siteinfo.pop('userinfo', None)
+	siteinfo.pop('userinfo', None)
 
-    siteinfo_doc = siteinfo_db.get(db_name)
+	siteinfo_doc = siteinfo_db.get(db_name)
 
-    if siteinfo_doc:
-        siteinfo_doc.update(siteinfo)
-    else:
-        siteinfo_doc = siteinfo
+	if siteinfo_doc:
+		siteinfo_doc.update(siteinfo)
+	else:
+		siteinfo_doc = siteinfo
 
-    siteinfo_db[db_name] = siteinfo_doc
+	siteinfo_db[db_name] = siteinfo_doc
 
 
 def parse_args():
-    argparser = argparse.ArgumentParser()
-    argparser.add_argument('site', nargs='?',
-                           help=('MediaWiki site to scrape (host name), '
-                                 'e.g. en.m.wikipedia.org'))
-    argparser.add_argument('--site-path', default='/w/',
-                           help=('MediaWiki site API path'
-                                 'Default: %(default)s'))
-    argparser.add_argument('--site-ext', default='.php',
-                           help=('MediaWiki site API script extension'
-                                 'Default: %(default)s'))
-    argparser.add_argument('-c', '--couch',
-                           help=('CouchDB server URL. '
-                                 'Default: %(default)s'),
-                           default='http://localhost:5984')
-    argparser.add_argument('--db',
-                           help=('CouchDB database name. '
-                                 'If not specified, the name will be '
-                                 'derived from Mediawiki host name.'),
-                           default=None)
-    argparser.add_argument('--titles', nargs='+',
-                           help=('Download article pages with '
-                                 'these names (titles). '
-                                 'It name starts with @ it is '
-                                 'interpreted as name of file containing titles, '
-                                 'one per line, utf8 encoded.'))
-    argparser.add_argument('--start',
-                           help=('Download all article pages '
-                                 'beginning with this name'))
-    argparser.add_argument('--changes-since',
-                           help=('Download all article pages '
-                                 'that change since specified time. '
-                                 'Timestamp format is yyyymmddhhmmss. '
-                                 'See https://www.mediawiki.org/wiki/Timestamp. '
-                                 'Hours, minutes and seconds can be omited'
-                             ))
-    argparser.add_argument('--recent-days', type=int, default=1,
-                           help=('Number of days to look back for recent changes'))
-    argparser.add_argument('--recent', action='store_true',
-                           help=('Download recently changed articles only'))
-    argparser.add_argument('--timeout',
-                           default=30.0,
-                           type=float,
-                           help=('Network communications timeout. '
-                                 'Default: %(default)ss'))
-    argparser.add_argument('-S', '--siteinfo-only', action='store_true',
-                           help=('Fetch or update siteinfo, then exit'))
-    argparser.add_argument('-r', '--resume', nargs='?',
-                           default='',
-                           metavar='SESSION ID',
-                           help=('Resume previous scrape session. '
-                                 'This relies on stats saved in '
-                                 'mwscrape database.'))
-    argparser.add_argument('--sessions-db-name',
-                           default='mwscrape',
-                           help=('Name of database where '
-                                 'session info is stored. '
-                                 'Default: %(default)s'))
-    argparser.add_argument('--desc',
-                           action='store_true',
-                           help=('Request all pages in descending order'))
-
-    argparser.add_argument('--delete-not-found',
-                           action='store_true',
-                           help=('Remove non-existing pages from the database'))
-
-    argparser.add_argument('--speed',
-                           type=int,
-                           choices=range(0, 6),
-                           default=0,
-                           help=('Scrape speed'))
-
-    argparser.add_argument('--delay',
-                           type=float,
-                           default=0,
-                           help=('Pause before requesting rendered article '
-                                 'for this many seconds. Default: %(default)s.'
-                                 'Some sites limit request rate so that even '
-                                 'single-threaded, request-at-a-time scrapes are too fast'
-                                 'and additional delay needs to be introduced'))
-
-    argparser.add_argument('--namespace',
-                           type=int,
-                           default=0,
-                           help=('ID of MediaWiki namespace to '
-                                 'scrape. Default: %(default)s'))
-
-
-    return argparser.parse_args()
+	argparser = argparse.ArgumentParser()
+	argparser.add_argument('site', nargs='?',
+						   help=('MediaWiki site to scrape (host name), '
+								 'e.g. en.m.wikipedia.org'))
+	argparser.add_argument('--site-path', default='/w/',
+						   help=('MediaWiki site API path'
+								 'Default: %(default)s'))
+	argparser.add_argument('--site-ext', default='.php',
+						   help=('MediaWiki site API script extension'
+								 'Default: %(default)s'))
+	argparser.add_argument('-c', '--couch',
+						   help=('CouchDB server URL. '
+								 'Default: %(default)s'),
+						   default='http://localhost:5984')
+	argparser.add_argument('--db',
+						   help=('CouchDB database name. '
+								 'If not specified, the name will be '
+								 'derived from Mediawiki host name.'),
+						   default=None)
+	argparser.add_argument('--titles', nargs='+',
+						   help=('Download article pages with '
+								 'these names (titles). '
+								 'It name starts with @ it is '
+								 'interpreted as name of file containing titles, '
+								 'one per line, utf8 encoded.'))
+	argparser.add_argument('--start',
+						   help=('Download all article pages '
+								 'beginning with this name'))
+	argparser.add_argument('--changes-since',
+						   help=('Download all article pages '
+								 'that change since specified time. '
+								 'Timestamp format is yyyymmddhhmmss. '
+								 'See https://www.mediawiki.org/wiki/Timestamp. '
+								 'Hours, minutes and seconds can be omited'
+							 ))
+	argparser.add_argument('--recent-days', type=int, default=1,
+						   help=('Number of days to look back for recent changes'))
+	argparser.add_argument('--recent', action='store_true',
+						   help=('Download recently changed articles only'))
+	argparser.add_argument('--timeout',
+						   default=30.0,
+						   type=float,
+						   help=('Network communications timeout. '
+								 'Default: %(default)ss'))
+	argparser.add_argument('-S', '--siteinfo-only', action='store_true',
+						   help=('Fetch or update siteinfo, then exit'))
+	argparser.add_argument('-r', '--resume', nargs='?',
+						   default='',
+						   metavar='SESSION ID',
+						   help=('Resume previous scrape session. '
+								 'This relies on stats saved in '
+								 'mwscrape database.'))
+	argparser.add_argument('--sessions-db-name',
+						   default='mwscrape',
+						   help=('Name of database where '
+								 'session info is stored. '
+								 'Default: %(default)s'))
+	argparser.add_argument('--desc',
+						   action='store_true',
+						   help=('Request all pages in descending order'))
+
+	argparser.add_argument('--delete-not-found',
+						   action='store_true',
+						   help=('Remove non-existing pages from the database'))
+
+	argparser.add_argument('--speed',
+						   type=int,
+						   choices=range(0, 6),
+						   default=0,
+						   help=('Scrape speed'))
+
+	argparser.add_argument('--delay',
+						   type=float,
+						   default=0,
+						   help=('Pause before requesting rendered article '
+								 'for this many seconds. Default: %(default)s.'
+								 'Some sites limit request rate so that even '
+								 'single-threaded, request-at-a-time scrapes are too fast'
+								 'and additional delay needs to be introduced'))
+
+	argparser.add_argument('--namespace',
+						   type=int,
+						   default=0,
+						   help=('ID of MediaWiki namespace to '
+								 'scrape. Default: %(default)s'))
+
+
+	return argparser.parse_args()
 
 
 SHOW_FUNC = r"""
@@ -179,367 +179,367 @@ def parse_args():
 {
   var r = /href="\/wiki\/(.*?)"/gi;
   var replace = function(match, p1, offset, string) {
-    return 'href="' + p1.replace(/_/g, ' ') + '"';
+	return 'href="' + p1.replace(/_/g, ' ') + '"';
   };
   return doc.parse.text['*'].replace(r, replace);
 }
 """
 
 def set_show_func(db, show_func=SHOW_FUNC, force=False):
-    design_doc = db.get('_design/w', {})
-    shows = design_doc.get('shows', {})
-    if force or not shows.get('html'):
-        shows['html'] = show_func
-        design_doc['shows'] = shows
-        db['_design/w'] = design_doc
+	design_doc = db.get('_design/w', {})
+	shows = design_doc.get('shows', {})
+	if force or not shows.get('html'):
+		shows['html'] = show_func
+		design_doc['shows'] = shows
+		db['_design/w'] = design_doc
 
 
 Redirect = namedtuple('Redirect', 'page fragment')
 
 def redirects_to(site, from_title):
-    """ Same as mwclient.page.Page.redirects_to except it returns page and fragment
-    in a named tuple instead of just target page
-    """
-    info = site.api('query', prop='pageprops', titles=from_title, redirects='')['query']
-    if 'redirects' in info:
-        for page in info['redirects']:
-            if page['from'] == from_title:
-                return Redirect(
-                    page=mwclient.page.Page(site, page['to']),
-                    fragment=page.get('tofragment', u'')
-                )
-        return None
-    else:
-        return None
+	""" Same as mwclient.page.Page.redirects_to except it returns page and fragment
+	in a named tuple instead of just target page
+	"""
+	info = site.api('query', prop='pageprops', titles=from_title, redirects='')['query']
+	if 'redirects' in info:
+		for page in info['redirects']:
+			if page['from'] == from_title:
+				return Redirect(
+					page=mwclient.page.Page(site, page['to']),
+					fragment=page.get('tofragment', u'')
+				)
+		return None
+	else:
+		return None
 
 
 def scheme_and_host(site_host):
-    p = urlparse.urlparse(site_host)
-    scheme = p.scheme if p.scheme else 'https'
-    host = p.netloc if p.scheme else site_host
-    return scheme, host
+	p = urlparse.urlparse(site_host)
+	scheme = p.scheme if p.scheme else 'https'
+	host = p.netloc if p.scheme else site_host
+	return scheme, host
 
 
 def mkcouch(url):
-    parsed = urlparse.urlparse(url)
-    server_url = parsed.scheme + '://'+ parsed.netloc
-    server = couchdb.Server(server_url)
-    user = parsed.username
-    password = parsed.password
-    if password:
-        print('Connecting %s as user %s' % (server.resource.url, user))
-        server.resource.credentials = (user, password)
-    return server
+	parsed = urlparse.urlparse(url)
+	server_url = parsed.scheme + '://'+ parsed.netloc
+	server = couchdb.Server(server_url)
+	user = parsed.username
+	password = parsed.password
+	if password:
+		print('Connecting %s as user %s' % (server.resource.url, user))
+		server.resource.credentials = (user, password)
+	return server
 
 
 @contextmanager
 def flock(path):
-    with open(path, 'w') as lock_fd:
-        try:
-            fcntl.flock(lock_fd, fcntl.LOCK_EX|fcntl.LOCK_NB)
-            yield
-        except IOError as ex:
-            if ex.errno == 11:
-                print (
-                    'Scrape for this host is already in progress. '
-                    'Use --speed option instead of starting multiple processes.')
-            raise SystemExit(1)
-        finally:
-            lock_fd.close()
+	with open(path, 'w') as lock_fd:
+		try:
+			fcntl.flock(lock_fd, fcntl.LOCK_EX|fcntl.LOCK_NB)
+			yield
+		except IOError as ex:
+			if ex.errno == 11:
+				print (
+					'Scrape for this host is already in progress. '
+					'Use --speed option instead of starting multiple processes.')
+			raise SystemExit(1)
+		finally:
+			lock_fd.close()
 
 
 def display_str(value, encoding='utf8'):
-    """
+	"""
 
-    >>> display_str(None)
-    ''
+	>>> display_str(None)
+	''
 
-    >>> display_str(u'\u0000')
-    '\\x00'
+	>>> display_str(u'\u0000')
+	'\\x00'
 
-    >>> display_str(u'\u044b', encoding='ascii')
-    "u'\\\\u044b'"
+	>>> display_str(u'\u044b', encoding='ascii')
+	"u'\\\\u044b'"
 
-    """
-    try:
-        value_str = '' if value is None else value.encode(encoding)
-    except UnicodeEncodeError:
-        value_str = repr(value).encode(encoding)
-    return value_str
+	"""
+	try:
+		value_str = '' if value is None else value.encode(encoding)
+	except UnicodeEncodeError:
+		value_str = repr(value).encode(encoding)
+	return value_str
 
 def fmt_mw_tms(dt):
-    return datetime.strftime(dt, '%Y%m%d%H%M%S')
+	return datetime.strftime(dt, '%Y%m%d%H%M%S')
 
 def main():
 
-    args = parse_args()
-
-    socket.setdefaulttimeout(args.timeout)
-
-    couch_server = mkcouch(args.couch)
-
-    sessions_db_name = args.sessions_db_name
-    try:
-        sessions_db = couch_server.create(sessions_db_name)
-    except couchdb.PreconditionFailed:
-        sessions_db = couch_server[sessions_db_name]
-
-    if args.resume or args.resume is None:
-        session_id = args.resume
-        if session_id is None:
-            current_doc = sessions_db['$current']
-            session_id = current_doc['session_id']
-        print('Resuming session %s' % session_id)
-        session_doc = sessions_db[session_id]
-        site_host = session_doc['site']
-        scheme, host = scheme_and_host(site_host)
-        db_name = session_doc['db_name']
-        session_doc['resumed_at'] = datetime.utcnow().isoformat()
-        if args.start:
-            start_page_name = args.start
-        else:
-            start_page_name = session_doc.get('last_page_name', args.start)
-        if args.desc:
-            descending = True
-        else:
-            descending = session_doc.get('descending', False)
-        sessions_db[session_id] = session_doc
-    else:
-        site_host = args.site
-        db_name = args.db
-        start_page_name = args.start
-        descending = args.desc
-        if not site_host:
-            print('Site to scrape is not specified')
-            raise SystemExit(1)
-        scheme, host = scheme_and_host(site_host)
-        if not db_name:
-            db_name = host.replace('.', '-')
-        session_id = '-'.join((db_name,
-                               str(int(time.time())),
-                               str(int(1000*random.random()))))
-        print('Starting session %s' % session_id)
-        sessions_db[session_id] = {
-            'created_at': datetime.utcnow().isoformat(),
-            'site': site_host,
-            'db_name': db_name,
-            'descending': descending
-        }
-        current_doc = sessions_db.get('$current', {})
-        current_doc['session_id'] = session_id
-        sessions_db['$current'] = current_doc
-
-
-    site = mwclient.Site((scheme, host), path=args.site_path, ext=args.site_ext)
-
-    update_siteinfo(site, couch_server, db_name)
-
-    if args.siteinfo_only:
-        return
-
-    try:
-        db = couch_server.create(db_name)
-    except couchdb.PreconditionFailed:
-        db = couch_server[db_name]
-
-    set_show_func(db)
-
-    def titles_from_args(titles):
-        for title in titles:
-            if title.startswith('@'):
-                with open(os.path.expanduser(title[1:])) as f:
-                    for line in f:
-                        yield line.strip()
-            else:
-                yield title
-
-    def recently_changed_pages(timestamp):
-        changes = site.recentchanges(start=timestamp,
-                                     namespace=0,
-                                     toponly=1,
-                                     type='edit|new',
-                                     dir='newer',
-                                     show='!minor|!redirect|!anon|!bot')
-        for page in changes:
-            title = page.get('title')
-            if title:
-                doc = db.get(title)
-                doc_revid = doc.get('parse', {}).get('revid') if doc else None
-                revid = page.get('revid')
-                if doc_revid == revid:
-                    continue
-                yield title
-
-    page_list = mwclient.listing.PageList(site, namespace=args.namespace)
-
-    if args.titles:
-        pages = (page_list[title.decode('utf8')]
-                 for title in titles_from_args(args.titles))
-    elif args.changes_since or args.recent:
-        if args.recent:
-            recent_days = args.recent_days
-            changes_since = fmt_mw_tms(datetime.utcnow() + timedelta(days=-recent_days))
-        else:
-            changes_since = args.changes_since.ljust(14, '0')
-        print('Getting recent changes (since %s)' % changes_since)
-        pages = (page_list[title]
-                 for title in recently_changed_pages(changes_since))
-
-    else:
-        print('Starting at %s' % start_page_name)
-        pages = site.allpages(start=start_page_name,
-                              namespace=args.namespace,
-                              dir='descending' if descending else 'ascending')
-
-    #threads are updating the same session document,
-    #we don't want to have conflicts
-    lock = RLock()
-
-    def inc_count(count_name):
-        with lock:
-            session_doc = sessions_db[session_id]
-            count = session_doc.get(count_name, 0)
-            session_doc[count_name] = count + 1
-            sessions_db[session_id] = session_doc
-
-    def update_session(title):
-        with lock:
-            session_doc = sessions_db[session_id]
-            session_doc['last_page_name'] = title
-            session_doc['updated_at'] = datetime.utcnow().isoformat()
-            sessions_db[session_id] = session_doc
-
-    def process(page):
-        title = page.name
-        if not page.exists:
-            title_str = display_str(title)
-            print('Not found: %s' % title_str)
-            inc_count('not_found')
-            if args.delete_not_found:
-                try:
-                    del db[title]
-                except couchdb.ResourceNotFound:
-                    print('%s was not in the database' % title_str)
-                except couchdb.ResourceConflict:
-                    print('Conflict while deleting %s' % title_str)
-                except Exception:
-                    traceback.print_exc()
-                else:
-                    print('%s removed from the database' % title_str)
-            return
-        try:
-            aliases = set()
-            redirect_count = 0
-            while page.redirect:
-                redirect_count += 1
-                redirect_target = redirects_to(site, page.name)
-                frag = redirect_target.fragment
-                if frag:
-                    alias = (title, frag)
-                else:
-                    alias = title
-                aliases.add(alias)
-
-                page = redirect_target.page
-                print('%s ==> %s' % (
-                    display_str(title),
-                    display_str(page.name) + (('#'+frag) if frag else '')))
-
-                if redirect_count >= 10:
-                    print('Too many redirect levels: %r' % aliases)
-                    break
-
-                title = page.name
-
-            if page.redirect:
-                print('Failed to resolve redirect %s', title)
-                inc_count('failed_redirect')
-                return
-
-            doc = db.get(title)
-            if doc:
-                current_aliases = set()
-                for alias in doc.get('aliases', ()):
-                    if isinstance(alias, list):
-                        alias = tuple(alias)
-                    current_aliases.add(alias)
-                if not aliases.issubset(current_aliases):
-                    merged_aliases = aliases|current_aliases
-                    #remove aliases without fragment if one with fragment is present
-                    #this is mostly to cleanup aliases in old scrapes
-                    to_remove = set()
-                    for alias in merged_aliases:
-                        if isinstance(alias, tuple):
-                            to_remove.add(alias[0])
-                    merged_aliases = merged_aliases - to_remove
-                    doc['aliases'] = list(merged_aliases)
-                    db[title] = doc
-                revid = doc.get('parse', {}).get('revid')
-                if page.revision == revid:
-                    print('%s is up to date (rev. %s), skipping' %
-                          (display_str(title), revid))
-                    inc_count('up_to_date')
-                    return
-                else:
-                    inc_count('updated')
-                    print('[%s] rev. %s => %s %s' %
-                          (time.strftime('%x %X', (page.touched))
-                           if page.touched else '?',
-                           revid, page.revision, display_str(title)))
-            if args.delay:
-                time.sleep(args.delay)
-            parse = site.api('parse', page=title)
-        except KeyboardInterrupt as ki:
-            print ('Caught KeyboardInterrupt', ki)
-            thread.interrupt_main()
-        except couchdb.ResourceConflict:
-            print('Update conflict, skipping: %s' % display_str(title))
-            return
-        except Exception:
-            print('Failed to process %s:' % display_str(title))
-            traceback.print_exc()
-            inc_count('error')
-            return
-        if doc:
-            doc.update(parse)
-        else:
-            inc_count('new')
-            doc = parse
-            if aliases:
-                doc['aliases'] = list(aliases)
-        try:
-            db[title] = doc
-        except couchdb.ResourceConflict:
-            print('Update conflict, skipping: %s' % display_str(title))
-            return
-
-    import pylru
-    seen = pylru.lrucache(10000)
-
-    def ipages(pages):
-        for index, page in enumerate(pages):
-            title = page.name
-            title_str = display_str(title)
-            print('%7s %s' % (index, title_str))
-            if title in seen:
-                print('Already saw %s, skipping' % (title_str,))
-                continue
-            seen[title] = True
-            update_session(title)
-            yield page
-
-
-    with flock(os.path.join(tempfile.gettempdir(),
-                            hashlib.sha1(host).hexdigest())):
-        if args.speed and not args.delay:
-            pool = ThreadPool(processes=args.speed*2)
-            for _result in pool.imap(process, ipages(pages)):
-                pass
-
-        else:
-            for page in ipages(pages):
-                process(page)
+	args = parse_args()
+
+	socket.setdefaulttimeout(args.timeout)
+
+	couch_server = mkcouch(args.couch)
+
+	sessions_db_name = args.sessions_db_name
+	try:
+		sessions_db = couch_server.create(sessions_db_name)
+	except couchdb.PreconditionFailed:
+		sessions_db = couch_server[sessions_db_name]
+
+	if args.resume or args.resume is None:
+		session_id = args.resume
+		if session_id is None:
+			current_doc = sessions_db['$current']
+			session_id = current_doc['session_id']
+		print('Resuming session %s' % session_id)
+		session_doc = sessions_db[session_id]
+		site_host = session_doc['site']
+		scheme, host = scheme_and_host(site_host)
+		db_name = session_doc['db_name']
+		session_doc['resumed_at'] = datetime.utcnow().isoformat()
+		if args.start:
+			start_page_name = args.start
+		else:
+			start_page_name = session_doc.get('last_page_name', args.start)
+		if args.desc:
+			descending = True
+		else:
+			descending = session_doc.get('descending', False)
+		sessions_db[session_id] = session_doc
+	else:
+		site_host = args.site
+		db_name = args.db
+		start_page_name = args.start
+		descending = args.desc
+		if not site_host:
+			print('Site to scrape is not specified')
+			raise SystemExit(1)
+		scheme, host = scheme_and_host(site_host)
+		if not db_name:
+			db_name = host.replace('.', '-')
+		session_id = '-'.join((db_name,
+							   str(int(time.time())),
+							   str(int(1000*random.random()))))
+		print('Starting session %s' % session_id)
+		sessions_db[session_id] = {
+			'created_at': datetime.utcnow().isoformat(),
+			'site': site_host,
+			'db_name': db_name,
+			'descending': descending
+		}
+		current_doc = sessions_db.get('$current', {})
+		current_doc['session_id'] = session_id
+		sessions_db['$current'] = current_doc
+
+
+	site = mwclient.Site((scheme, host), path=args.site_path, ext=args.site_ext)
+
+	update_siteinfo(site, couch_server, db_name)
+
+	if args.siteinfo_only:
+		return
+
+	try:
+		db = couch_server.create(db_name)
+	except couchdb.PreconditionFailed:
+		db = couch_server[db_name]
+
+	set_show_func(db)
+
+	def titles_from_args(titles):
+		for title in titles:
+			if title.startswith('@'):
+				with open(os.path.expanduser(title[1:])) as f:
+					for line in f:
+						yield line.strip()
+			else:
+				yield title
+
+	def recently_changed_pages(timestamp):
+		changes = site.recentchanges(start=timestamp,
+									 namespace=0,
+									 toponly=1,
+									 type='edit|new',
+									 dir='newer',
+									 show='!minor|!redirect|!anon|!bot')
+		for page in changes:
+			title = page.get('title')
+			if title:
+				doc = db.get(title)
+				doc_revid = doc.get('parse', {}).get('revid') if doc else None
+				revid = page.get('revid')
+				if doc_revid == revid:
+					continue
+				yield title
+
+	page_list = mwclient.listing.PageList(site, namespace=args.namespace)
+
+	if args.titles:
+		pages = (page_list[title.decode('utf8')]
+				 for title in titles_from_args(args.titles))
+	elif args.changes_since or args.recent:
+		if args.recent:
+			recent_days = args.recent_days
+			changes_since = fmt_mw_tms(datetime.utcnow() + timedelta(days=-recent_days))
+		else:
+			changes_since = args.changes_since.ljust(14, '0')
+		print('Getting recent changes (since %s)' % changes_since)
+		pages = (page_list[title]
+				 for title in recently_changed_pages(changes_since))
+
+	else:
+		print('Starting at %s' % start_page_name)
+		pages = site.allpages(start=start_page_name,
+							  namespace=args.namespace,
+							  dir='descending' if descending else 'ascending')
+
+	#threads are updating the same session document,
+	#we don't want to have conflicts
+	lock = RLock()
+
+	def inc_count(count_name):
+		with lock:
+			session_doc = sessions_db[session_id]
+			count = session_doc.get(count_name, 0)
+			session_doc[count_name] = count + 1
+			sessions_db[session_id] = session_doc
+
+	def update_session(title):
+		with lock:
+			session_doc = sessions_db[session_id]
+			session_doc['last_page_name'] = title
+			session_doc['updated_at'] = datetime.utcnow().isoformat()
+			sessions_db[session_id] = session_doc
+
+	def process(page):
+		title = page.name
+		if not page.exists:
+			title_str = display_str(title)
+			print('Not found: %s' % title_str)
+			inc_count('not_found')
+			if args.delete_not_found:
+				try:
+					del db[title]
+				except couchdb.ResourceNotFound:
+					print('%s was not in the database' % title_str)
+				except couchdb.ResourceConflict:
+					print('Conflict while deleting %s' % title_str)
+				except Exception:
+					traceback.print_exc()
+				else:
+					print('%s removed from the database' % title_str)
+			return
+		try:
+			aliases = set()
+			redirect_count = 0
+			while page.redirect:
+				redirect_count += 1
+				redirect_target = redirects_to(site, page.name)
+				frag = redirect_target.fragment
+				if frag:
+					alias = (title, frag)
+				else:
+					alias = title
+				aliases.add(alias)
+
+				page = redirect_target.page
+				print('%s ==> %s' % (
+					display_str(title),
+					display_str(page.name) + (('#'+frag) if frag else '')))
+
+				if redirect_count >= 10:
+					print('Too many redirect levels: %r' % aliases)
+					break
+
+				title = page.name
+
+			if page.redirect:
+				print('Failed to resolve redirect %s', title)
+				inc_count('failed_redirect')
+				return
+
+			doc = db.get(title)
+			if doc:
+				current_aliases = set()
+				for alias in doc.get('aliases', ()):
+					if isinstance(alias, list):
+						alias = tuple(alias)
+					current_aliases.add(alias)
+				if not aliases.issubset(current_aliases):
+					merged_aliases = aliases|current_aliases
+					#remove aliases without fragment if one with fragment is present
+					#this is mostly to cleanup aliases in old scrapes
+					to_remove = set()
+					for alias in merged_aliases:
+						if isinstance(alias, tuple):
+							to_remove.add(alias[0])
+					merged_aliases = merged_aliases - to_remove
+					doc['aliases'] = list(merged_aliases)
+					db[title] = doc
+				revid = doc.get('parse', {}).get('revid')
+				if page.revision == revid:
+					print('%s is up to date (rev. %s), skipping' %
+						  (display_str(title), revid))
+					inc_count('up_to_date')
+					return
+				else:
+					inc_count('updated')
+					print('[%s] rev. %s => %s %s' %
+						  (time.strftime('%x %X', (page.touched))
+						   if page.touched else '?',
+						   revid, page.revision, display_str(title)))
+			if args.delay:
+				time.sleep(args.delay)
+			parse = site.api('parse', page=title)
+		except KeyboardInterrupt as ki:
+			print ('Caught KeyboardInterrupt', ki)
+			thread.interrupt_main()
+		except couchdb.ResourceConflict:
+			print('Update conflict, skipping: %s' % display_str(title))
+			return
+		except Exception:
+			print('Failed to process %s:' % display_str(title))
+			traceback.print_exc()
+			inc_count('error')
+			return
+		if doc:
+			doc.update(parse)
+		else:
+			inc_count('new')
+			doc = parse
+			if aliases:
+				doc['aliases'] = list(aliases)
+		try:
+			db[title] = doc
+		except couchdb.ResourceConflict:
+			print('Update conflict, skipping: %s' % display_str(title))
+			return
+
+	import pylru
+	seen = pylru.lrucache(10000)
+
+	def ipages(pages):
+		for index, page in enumerate(pages):
+			title = page.name
+			title_str = display_str(title)
+			print('%7s %s' % (index, title_str))
+			if title in seen:
+				print('Already saw %s, skipping' % (title_str,))
+				continue
+			seen[title] = True
+			update_session(title)
+			yield page
+
+
+	with flock(os.path.join(tempfile.gettempdir(),
+							hashlib.sha1(host).hexdigest())):
+		if args.speed and not args.delay:
+			pool = ThreadPool(processes=args.speed*2)
+			for _result in pool.imap(process, ipages(pages)):
+				pass
+
+		else:
+			for page in ipages(pages):
+				process(page)
 
 
 if __name__ == '__main__':
-    main()
+	main()

From fb13c29ddc1d3266cf9c407d26271d87cfeab871 Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Fri, 12 Apr 2019 03:36:58 +1000
Subject: [PATCH 03/12] conv to python 3 halfway

---
 mwscrape/scrape.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/mwscrape/scrape.py b/mwscrape/scrape.py
index e3e5355..99e7608 100644
--- a/mwscrape/scrape.py
+++ b/mwscrape/scrape.py
@@ -12,10 +12,10 @@
 import os
 import socket
 import traceback
-import urlparse
+from urllib.parse import urlparse
 import tempfile
 import time
-import thread
+import _thread
 import random
 
 from collections import namedtuple
@@ -50,7 +50,7 @@ def fix_server_url(general_siteinfo):
 	"""
 	server = general_siteinfo.get('server', '')
 	if server:
-		p = urlparse.urlparse(server)
+		p = urlparse(server)
 		if not p.scheme:
 			server = urlparse.urlunparse(
 				urlparse.ParseResult('http', p.netloc, p.path,
@@ -214,14 +214,14 @@ def redirects_to(site, from_title):
 
 
 def scheme_and_host(site_host):
-	p = urlparse.urlparse(site_host)
+	p = urlparse(site_host)
 	scheme = p.scheme if p.scheme else 'https'
 	host = p.netloc if p.scheme else site_host
 	return scheme, host
 
 
 def mkcouch(url):
-	parsed = urlparse.urlparse(url)
+	parsed = urlparse(url)
 	server_url = parsed.scheme + '://'+ parsed.netloc
 	server = couchdb.Server(server_url)
 	user = parsed.username
@@ -236,9 +236,9 @@ def mkcouch(url):
 def flock(path):
 	with open(path, 'w') as lock_fd:
 		try:
-			fcntl.flock(lock_fd, fcntl.LOCK_EX|fcntl.LOCK_NB)
+			FileLock(lock_fd)
 			yield
-		except IOError as ex:
+		except Exception as ex:
 			if ex.errno == 11:
 				print (
 					'Scrape for this host is already in progress. '
@@ -491,7 +491,7 @@ def process(page):
 			parse = site.api('parse', page=title)
 		except KeyboardInterrupt as ki:
 			print ('Caught KeyboardInterrupt', ki)
-			thread.interrupt_main()
+			__thread.interrupt_main()
 		except couchdb.ResourceConflict:
 			print('Update conflict, skipping: %s' % display_str(title))
 			return

From 5edef48f941d93c6d33637ebf0d0856885fe33e1 Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Fri, 12 Apr 2019 04:10:25 +1000
Subject: [PATCH 04/12] mov everything to one dir

---
 mwscrape/__init__.py => __init__.py                 | 0
 mwscrape/resolveconflicts.py => resolveconflicts.py | 0
 mwscrape/scrape.py => scrape.py                     | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename mwscrape/__init__.py => __init__.py (100%)
 rename mwscrape/resolveconflicts.py => resolveconflicts.py (100%)
 rename mwscrape/scrape.py => scrape.py (100%)

diff --git a/mwscrape/__init__.py b/__init__.py
similarity index 100%
rename from mwscrape/__init__.py
rename to __init__.py
diff --git a/mwscrape/resolveconflicts.py b/resolveconflicts.py
similarity index 100%
rename from mwscrape/resolveconflicts.py
rename to resolveconflicts.py
diff --git a/mwscrape/scrape.py b/scrape.py
similarity index 100%
rename from mwscrape/scrape.py
rename to scrape.py

From 4706c89ad16de1bbf1c926248e3e9b6f827cad59 Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Fri, 12 Apr 2019 04:28:40 +1000
Subject: [PATCH 05/12] adapt further for PY3

---
 scrape.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrape.py b/scrape.py
index 99e7608..bdaa974 100644
--- a/scrape.py
+++ b/scrape.py
@@ -433,7 +433,7 @@ def process(page):
 			while page.redirect:
 				redirect_count += 1
 				redirect_target = redirects_to(site, page.name)
-				frag = redirect_target.fragment
+				frag = redirect_target.fragment.decode("utf-8")
 				if frag:
 					alias = (title, frag)
 				else:
@@ -530,7 +530,7 @@ def ipages(pages):
 
 
 	with flock(os.path.join(tempfile.gettempdir(),
-							hashlib.sha1(host).hexdigest())):
+							hashlib.sha1(host.encode('utf-8')).hexdigest())):
 		if args.speed and not args.delay:
 			pool = ThreadPool(processes=args.speed*2)
 			for _result in pool.imap(process, ipages(pages)):

From 20da02664a44d681d8019751a6cc4df6d45352de Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Fri, 12 Apr 2019 04:56:01 +1000
Subject: [PATCH 06/12] (I think) final fixes for PY3

---
 scrape.py | 57 ++++++++++++++++++++++++-------------------------------
 1 file changed, 25 insertions(+), 32 deletions(-)

diff --git a/scrape.py b/scrape.py
index bdaa974..6262b31 100644
--- a/scrape.py
+++ b/scrape.py
@@ -17,6 +17,8 @@
 import time
 import _thread
 import random
+import pylru
+
 
 from collections import namedtuple
 from datetime import datetime, timedelta
@@ -26,8 +28,6 @@
 
 import couchdb
 import mwclient
-import mwclient.page
-
 
 def fix_server_url(general_siteinfo):
 	"""
@@ -234,21 +234,9 @@ def mkcouch(url):
 
 @contextmanager
 def flock(path):
-	with open(path, 'w') as lock_fd:
-		try:
-			FileLock(lock_fd)
-			yield
-		except Exception as ex:
-			if ex.errno == 11:
-				print (
-					'Scrape for this host is already in progress. '
-					'Use --speed option instead of starting multiple processes.')
-			raise SystemExit(1)
-		finally:
-			lock_fd.close()
+	FileLock(path)
 
-
-def display_str(value, encoding='utf8'):
+def display_str(value, encoding='utf-8'):
 	"""
 
 	>>> display_str(None)
@@ -262,9 +250,10 @@ def display_str(value, encoding='utf8'):
 
 	"""
 	try:
-		value_str = '' if value is None else value.encode(encoding)
+		value_str = '' if value is None else value.encode(encoding).decode(encoding)
 	except UnicodeEncodeError:
 		value_str = repr(value).encode(encoding)
+		
 	return value_str
 
 def fmt_mw_tms(dt):
@@ -433,7 +422,8 @@ def process(page):
 			while page.redirect:
 				redirect_count += 1
 				redirect_target = redirects_to(site, page.name)
-				frag = redirect_target.fragment.decode("utf-8")
+				frag = redirect_target.fragment
+				
 				if frag:
 					alias = (title, frag)
 				else:
@@ -443,7 +433,7 @@ def process(page):
 				page = redirect_target.page
 				print('%s ==> %s' % (
 					display_str(title),
-					display_str(page.name) + (('#'+frag) if frag else '')))
+					display_str(page.name) + (("#"+frag) if frag else '')))
 
 				if redirect_count >= 10:
 					print('Too many redirect levels: %r' % aliases)
@@ -491,7 +481,7 @@ def process(page):
 			parse = site.api('parse', page=title)
 		except KeyboardInterrupt as ki:
 			print ('Caught KeyboardInterrupt', ki)
-			__thread.interrupt_main()
+			_thread.interrupt_main()
 		except couchdb.ResourceConflict:
 			print('Update conflict, skipping: %s' % display_str(title))
 			return
@@ -513,14 +503,13 @@ def process(page):
 			print('Update conflict, skipping: %s' % display_str(title))
 			return
 
-	import pylru
 	seen = pylru.lrucache(10000)
 
 	def ipages(pages):
 		for index, page in enumerate(pages):
 			title = page.name
 			title_str = display_str(title)
-			print('%7s %s' % (index, title_str))
+			print('%7s. %s' % (index, title_str))
 			if title in seen:
 				print('Already saw %s, skipping' % (title_str,))
 				continue
@@ -528,17 +517,21 @@ def ipages(pages):
 			update_session(title)
 			yield page
 
+	buf = os.path.join(
+		tempfile.gettempdir(),
+		hashlib.sha1(host.encode('utf-8')).hexdigest()
+	)
+	
+	flock(buf)
+	
+	if args.speed and not args.delay:
+		pool = ThreadPool(processes=args.speed*2)
+		for _result in pool.imap(process, ipages(pages)):
+			pass
 
-	with flock(os.path.join(tempfile.gettempdir(),
-							hashlib.sha1(host.encode('utf-8')).hexdigest())):
-		if args.speed and not args.delay:
-			pool = ThreadPool(processes=args.speed*2)
-			for _result in pool.imap(process, ipages(pages)):
-				pass
-
-		else:
-			for page in ipages(pages):
-				process(page)
+	else:
+		for page in ipages(pages):
+			process(page)
 
 
 if __name__ == '__main__':

From ce83294c224151889f59837c143238d66f8b3ccb Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Wed, 24 Apr 2019 20:29:11 +1000
Subject: [PATCH 07/12] move everything to root dir, as normally

---
 scrape.py => mwscrape.py | 0
 setup.py                 | 4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename scrape.py => mwscrape.py (100%)

diff --git a/scrape.py b/mwscrape.py
similarity index 100%
rename from scrape.py
rename to mwscrape.py
diff --git a/setup.py b/setup.py
index df0a805..615240e 100644
--- a/setup.py
+++ b/setup.py
@@ -11,6 +11,6 @@
       #mwclient appears to need six, but doesn't declare it as dependency
       install_requires=['futures', 'CouchDB >= 0.10', 'mwclient >= 0.7.2', 'pylru'],
       entry_points={'console_scripts': [
-          'mwscrape=mwscrape.scrape:main',
-          'mwresolvec=mwscrape.resolveconflicts:main',
+          'mwscrape=mwscrape:main',
+          'mwresolvec=resolveconflicts:main',
       ]})

From 6e3f051eb3da8cd17bd59e721e820045917051db Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Thu, 25 Apr 2019 21:26:49 +1000
Subject: [PATCH 08/12] ignore dicts happened to be created in same dir

---
 .gitignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 54e9d64..ec7d824 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 *.pyc
 *~
 *.egg-info
+*.slob
 __pycache__
-README.html
\ No newline at end of file
+README.html

From 607d9edb2913f3a0a6a34951c9963c38dfd55add Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Thu, 25 Apr 2019 21:38:38 +1000
Subject: [PATCH 09/12] small fixes

---
 __init__.py      | 0
 requirements.txt | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 __init__.py

diff --git a/__init__.py b/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/requirements.txt b/requirements.txt
index 666ac7e..43adc6c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@ futures
 CouchDB >= 0.10'
 mwclient >= 0.7.2'
 pylru
-filelock
\ No newline at end of file
+filelock

From 65fa11a98b661279b0fada323f3756cdd76ecc23 Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Thu, 25 Apr 2019 21:43:27 +1000
Subject: [PATCH 10/12] small fixes (2)

---
 requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 43adc6c..d780a2e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 futures
-CouchDB >= 0.10'
-mwclient >= 0.7.2'
+CouchDB >= 0.10
+mwclient >= 0.7.2
 pylru
 filelock

From 1feaab3de662b4c6445214fd796626c404f42daa Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Fri, 26 Apr 2019 23:06:26 +1000
Subject: [PATCH 11/12] refresh setup.py

---
 .gitignore |  1 +
 setup.py   | 48 ++++++++++++++++++++++++++++++++++--------------
 2 files changed, 35 insertions(+), 14 deletions(-)

diff --git a/.gitignore b/.gitignore
index ec7d824..d9a57b3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@
 *.slob
 __pycache__
 README.html
+*.zip
diff --git a/setup.py b/setup.py
index 615240e..554b91c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,16 +1,36 @@
 from distutils.core import setup
 
-setup(name='mwscrape',
-      version='1.0',
-      description='Download',
-      author='Igor Tkach',
-      author_email='itkach@gmail.com',
-      url='http://github.com/itkach/mwscrape',
-      license='MPL 2.0',
-      packages=['mwscrape'],
-      #mwclient appears to need six, but doesn't declare it as dependency
-      install_requires=['futures', 'CouchDB >= 0.10', 'mwclient >= 0.7.2', 'pylru'],
-      entry_points={'console_scripts': [
-          'mwscrape=mwscrape:main',
-          'mwresolvec=resolveconflicts:main',
-      ]})
+setup(
+		name='mwscrape',
+		version='1.1',
+		description='Download',
+		author='Igor Tkach',
+		author_email='itkach@gmail.com',
+		url='http://github.com/itkach/mwscrape',
+		license='MPL 2.0',
+		#packages=['mwscrape'],
+		#package_dir={'': '..'},
+		#mwclient appears to need six, but doesn't declare it as dependency
+		install_requires=['futures', 'CouchDB >= 0.10', 'mwclient >= 0.7.2', 'pylru'],
+		entry_points={'console_scripts': [
+			'mwscrape=mwscrape:main',
+			'mwresolvec=resolveconflicts:main',
+		]},
+		
+		classifiers=[
+			'Development Status :: 5 - Production/Stable',
+			
+			'Intended Audience :: Developers',
+			'Intended Audience :: SEO specialists',
+			'Intended Audience :: Information Technology',
+			'Intended Audience :: Science/Research',
+			
+			'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
+			'Operating System :: OS Independent',
+			
+			'Programming Language :: Python',
+			'Programming Language :: Python :: 3',
+			'Programming Language :: Python :: 3.7',
+			'Topic :: Internet :: WWW/HTTP :: Indexing/Search',
+		],
+)
\ No newline at end of file

From 7451bb57ad5fdd0498ab73ee0d9d9a1ff9edee23 Mon Sep 17 00:00:00 2001
From: Tim <TAbdiukov@gmail.com>
Date: Fri, 26 Apr 2019 23:14:51 +1000
Subject: [PATCH 12/12] prep setup.py for script publishing

---
 setup.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 554b91c..6d897bf 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,13 @@
 from distutils.core import setup
 
+import os
+import sys
+
+if 'publish' in sys.argv:
+    os.system('python setup.py sdist upload')
+    sys.exit()
+
+
 setup(
 		name='mwscrape',
 		version='1.1',
@@ -8,8 +16,10 @@
 		author_email='itkach@gmail.com',
 		url='http://github.com/itkach/mwscrape',
 		license='MPL 2.0',
-		#packages=['mwscrape'],
+		
+		#packages=['mwscrape'], # don't have to have this to create a package
 		#package_dir={'': '..'},
+		
 		#mwclient appears to need six, but doesn't declare it as dependency
 		install_requires=['futures', 'CouchDB >= 0.10', 'mwclient >= 0.7.2', 'pylru'],
 		entry_points={'console_scripts': [