- to 0.8.33 and update mounting functionality [1]

- pet portlint, don't bring in GETTEXT if defined (WITHOUT_NLS) [2]

PR:		ports/164215 [1]
Submitted by:	maintainer [1], scheidell [2]
Approved by:	gabor (mentor)

6 files changed, 505 insertions, 96 deletions

diff --git a/deskutils/calibre/Makefile b/deskutils/calibre/Makefile
index 8fac1a179445..0765c9511434 100644
--- a/deskutils/calibre/Makefile
+++ b/deskutils/calibre/Makefile
@@ -6,7 +6,7 @@
 #
 
 PORTNAME=	calibre
-PORTVERSION=	0.8.26
+PORTVERSION=	0.8.33
 CATEGORIES=	deskutils python
 MASTER_SITES=	SF/${PORTNAME}/${PORTVERSION}/
 
@@ -39,12 +39,18 @@ RUN_DEPENDS=	xdg-open:${PORTSDIR}/devel/xdg-utils \
 		py*-qt4-network>=0:${PORTSDIR}/net/py-qt4-network \
 		py*-qt4-svg>=0:${PORTSDIR}/graphics/py-qt4-svg \
 		py*-qt4-webkit>=0:${PORTSDIR}/www/py-qt4-webkit \
-		py*-qt4-xml>=0:${PORTSDIR}/textproc/py-qt4-xml
+		py*-qt4-xml>=0:${PORTSDIR}/textproc/py-qt4-xml \
+		py*-dbus>=0:${PORTSDIR}/devel/py-dbus \
+		pdftohtml:${PORTSDIR}/graphics/poppler-utils \
+		lshal:${PORTSDIR}/sysutils/hal
 
-OPTIONS=	PODOFO "PDF support via PoDoFo library" on \
-		MOUNTHELPER_SUID "Install mount helper as suid binary" off
+OPTIONS=	PODOFO	"PDF support via PoDoFo library" on \
+		NLS	"Enable NLS support"	on
 
+USE_XZ=		yes
+.if defined(WITH_NLS)
 USE_GETTEXT=	yes
+.endif
 USE_PYTHON=	2.7+
 USE_QT_VER=	4
 QT_COMPONENTS=	qmake_build moc_build
@@ -101,12 +107,9 @@ do-install:
 	${MKDIR} ${PREFIX}/share/mime/packages
 	${MKDIR} ${PREFIX}/share/icons/hicolor/128x128/apps
 	@(cd ${BUILD_WRKSRC}; ${SETENV} ${MAKE_ENV} ${PYTHON_CMD} \
-		${PYSETUP} install --prefix ${PREFIX})
+		${PYSETUP} ${INSTALL} --prefix ${PREFIX})
 
 post-install:
-.if defined(WITH_MOUNTHELPER_SUID)
-	${CHOWN} ${BINOWN}:operator ${PREFIX}/bin/calibre-mount-helper
-	${CHMOD} ug+s ${PREFIX}/bin/calibre-mount-helper
-.endif
+	@${CAT} ${PKGMESSAGE}
 
 .include <bsd.port.post.mk>
diff --git a/deskutils/calibre/distinfo b/deskutils/calibre/distinfo
index 296053d67679..e6e792056c78 100644
--- a/deskutils/calibre/distinfo
+++ b/deskutils/calibre/distinfo
@@ -1,2 +1,2 @@
-SHA256 (calibre-0.8.26.tar.gz) = 59e713e126ddb5739bfe33acf52581ccad944a9f05c3e1d3d0b3958b4762f91a
-SIZE (calibre-0.8.26.tar.gz) = 37784724
+SHA256 (calibre-0.8.33.tar.xz) = 885baed4810a92ceea78c49e4780a00aed19166b2475b430c9491396d3c0b74d
+SIZE (calibre-0.8.33.tar.xz) = 21848976
diff --git a/deskutils/calibre/files/patch-src_calibre_devices_scanner.py b/deskutils/calibre/files/patch-src_calibre_devices_scanner.py
index 1f8c598cabf5..182cbda0b879 100644
--- a/deskutils/calibre/files/patch-src_calibre_devices_scanner.py
+++ b/deskutils/calibre/files/patch-src_calibre_devices_scanner.py
@@ -1,11 +1,7 @@
---- src/calibre/devices/scanner.py.orig	2011-06-14 14:10:28.000000000 -0500
-+++ src/calibre/devices/scanner.py	2011-06-14 14:12:39.000000000 -0500
-@@ -5,10 +5,10 @@
- manner.
- '''
- 
--import sys, os, re
-+import sys, os, re, subprocess
+--- src/calibre/devices/scanner.py.orig	2011-11-11 20:22:34.000000000 -0800
++++ src/calibre/devices/scanner.py	2011-12-30 16:32:32.000000000 -0800
+@@ -8,7 +8,7 @@
+ import sys, os, re
  from threading import RLock
  
 -from calibre.constants import iswindows, isosx, plugins, islinux
@@ -13,7 +9,7 @@
  
  osx_scanner = win_scanner = linux_scanner = None
  
-@@ -155,17 +155,68 @@
+@@ -155,17 +155,80 @@
              ans.add(tuple(dev))
          return ans
  
@@ -21,43 +17,55 @@
 +
 +    def __call__(self):
 +        ans = set([])
++        import dbus
++        devs = []
 +
 +        try:
-+            out = subprocess.Popen("/usr/sbin/usbconfig dump_device_desc | /usr/bin/awk 'function get_str(s) { split(s, a, /<|>/); if (a[2] != \"no string\") { return a[2]; } else { return \"\";} } BEGIN {state=0;} /^[[:space:]]+idVendor/ {state = 1; vendor = $3; next;} /idProduct/ {productid = $3; next;} /bcdDevice/ {bcd = $3; next;} /iManufacturer/ { manufacturer = get_str($0); next; }  /iProduct/ { product = get_str($0); next;} /iSerialNumber/ { sn = get_str($0); next;} /^$/ {if (state == 1) { state = 0; printf(\"%s%%%%%s%%%%%s%%%%%s%%%%%s%%%%%s\\n\",vendor, productid, bcd, manufacturer, product, sn);} }'", shell=True, stdout=subprocess.PIPE).communicate()[0]
-+        except OSError, e:
++           bus = dbus.SystemBus()
++           manager = dbus.Interface(bus.get_object('org.freedesktop.Hal',
++                         '/org/freedesktop/Hal/Manager'), 'org.freedesktop.Hal.Manager')
++           paths = manager.FindDeviceStringMatch('freebsd.driver','da')
++           for path in paths:
++              obj = bus.get_object('org.freedesktop.Hal', path)
++              objif = dbus.Interface(obj, 'org.freedesktop.Hal.Device')
++              devif = objif
++              parentdriver = None
++              while parentdriver != 'umass':
++                 try:
++                    obj = bus.get_object('org.freedesktop.Hal',
++                          objif.GetProperty('info.parent'))
++                    objif = dbus.Interface(obj, 'org.freedesktop.Hal.Device')
++                    try:
++                       parentdriver = objif.GetProperty('freebsd.driver')
++                    except dbus.exceptions.DBusException, e:
++                       continue
++                 except dbus.exceptions.DBusException, e:
++                    break
++              if parentdriver != 'umass':
++                  continue
++              dev = []
++              try:
++                 dev.append(objif.GetProperty('usb.vendor_id'))
++                 dev.append(objif.GetProperty('usb.product_id'))
++                 dev.append(objif.GetProperty('usb.device_revision_bcd'))
++              except dbus.exceptions.DBusException, e:
++                 continue
++              try:
++                 dev.append(objif.GetProperty('info.vendor'))
++              except:
++                 dev.append('')
++              try:
++                 dev.append(objif.GetProperty('info.product'))
++              except:
++                 dev.append('')
++              try:
++                 dev.append(objif.GetProperty('usb.serial'))
++              except:
++                 dev.append('')
++              dev.append(path)
++              ans.add(tuple(dev))
++        except dbus.exceptions.DBusException, e:
 +           print >>sys.stderr, "Execution failed:", e
-+
-+        if out.strip() == "":
-+            return ans
-+        for line in out.strip().split("\n"):
-+            ven, prod, bcd, man, prod_string, serial = line.strip().split("%%", 6)
-+            dev = []
-+            try:
-+                dev.append(int(ven, 16))
-+            except:
-+                continue
-+            try:
-+                dev.append(int(prod, 16))
-+            except:
-+                continue
-+            try:
-+                dev.append(int(bcd, 16))
-+            except:
-+                continue
-+            try:
-+                dev.append(man)
-+            except:
-+                dev.append('')
-+            try:
-+                dev.append(prod_string)
-+            except:
-+                dev.append('')
-+            try:
-+                dev.append(serial)
-+            except:
-+                dev.append('')
-+
-+            ans.add(tuple(dev))
 +        return ans
 +
 +
diff --git a/deskutils/calibre/files/patch-src_calibre_devices_usbms_device.py b/deskutils/calibre/files/patch-src_calibre_devices_usbms_device.py
new file mode 100644
index 000000000000..bb342c395203
--- /dev/null
+++ b/deskutils/calibre/files/patch-src_calibre_devices_usbms_device.py
@@ -0,0 +1,299 @@
+--- src/calibre/devices/usbms/device.py.orig	2011-12-29 20:00:38.000000000 -0800
++++ src/calibre/devices/usbms/device.py	2012-01-01 14:09:00.000000000 -0800
+@@ -697,19 +697,21 @@
+             self._card_a_prefix = self._card_b_prefix
+             self._card_b_prefix = None
+ 
++
+ # ------------------------------------------------------
+ #
+ #  open for FreeBSD
+-#   find the device node or nodes that match the S/N we already have from the scanner
+-#   and attempt to mount each one
+-#       1.  get list of disk devices from sysctl
+-#       2.  compare that list with the one from camcontrol
+-#       3.  and see if it has a matching s/n
+-#       6.  find any partitions/slices associated with each node
+-#       7.  attempt to mount, using calibre-mount-helper, each one
+-#       8.  when finished, we have a list of mount points and associated device nodes
++#      find the device node or nodes that match the S/N we already have from the scanner
++#      and attempt to mount each one
++#              1.  get list of devices in /dev with matching s/n etc.
++#              2.  get list of volumes associated with each
++#              3.  attempt to mount each one using Hal
++#              4.  when finished, we have a list of mount points and associated dbus nodes
+ #
+     def open_freebsd(self):
++        import dbus
++        # There should be some way to access the -v arg...
++        verbose = False
+ 
+         # this gives us access to the S/N, etc. of the reader that the scanner has found
+         # and the match routines for some of that data, like s/n, vendor ID, etc.
+@@ -718,129 +720,149 @@
+         if not d.serial:
+             raise DeviceError("Device has no S/N.  Can't continue")
+             return False
+-
+-        devs={}
+-        di=0
+-        ndevs=4     # number of possible devices per reader (main, carda, cardb, launcher)
+-
+-        #get list of disk devices
+-        p=subprocess.Popen(["sysctl", "kern.disks"], stdout=subprocess.PIPE)
+-        kdsks=subprocess.Popen(["sed", "s/kern.disks: //"], stdin=p.stdout, stdout=subprocess.PIPE).communicate()[0]
+-        p.stdout.close()
+-        #print kdsks
+-        for dvc in kdsks.split():
+-            # for each one that's also in the list of cam devices ...
+-            p=subprocess.Popen(["camcontrol", "devlist"], stdout=subprocess.PIPE)
+-            devmatch=subprocess.Popen(["grep", dvc], stdin=p.stdout, stdout=subprocess.PIPE).communicate()[0]
+-            p.stdout.close()
+-            if devmatch:
+-                #print "Checking ", devmatch
+-                # ... see if we can get a S/N from the actual device node
+-                sn=subprocess.Popen(["camcontrol", "inquiry", dvc, "-S"], stdout=subprocess.PIPE).communicate()[0]
+-                sn=sn[0:-1]             # drop the trailing newline
+-                #print "S/N = ", sn
+-                if sn and d.match_serial(sn):
+-                    # we have a matching s/n, record this device node
+-                    #print "match found: ", dvc
+-                    devs[di]=dvc
+-                    di += 1
+-
+-        # sort the list of devices
+-        for i in range(1,ndevs+1):
+-            for j in reversed(range(1,i)):
+-                if devs[j-1] > devs[j]:
+-                    x=devs[j-1]
+-                    devs[j-1]=devs[j]
+-                    devs[j]=x
+-        #print devs
+-
+-        # now we need to see if any of these have slices/partitions
++        
++        vols=[]
++        
++        bus = dbus.SystemBus()
++        manager = dbus.Interface(bus.get_object('org.freedesktop.Hal',
++                      '/org/freedesktop/Hal/Manager'), 'org.freedesktop.Hal.Manager')
++        paths = manager.FindDeviceStringMatch('usb.serial',d.serial)
++        for path in paths:
++            objif = dbus.Interface(bus.get_object('org.freedesktop.Hal', path), 'org.freedesktop.Hal.Device')
++            # Extra paranoia...
++            try:
++                if d.idVendor == objif.GetProperty('usb.vendor_id') and \
++                        d.idProduct == objif.GetProperty('usb.product_id') and \
++                        d.manufacturer == objif.GetProperty('usb.vendor') and \
++                        d.product == objif.GetProperty('usb.product') and \
++                        d.serial == objif.GetProperty('usb.serial'):
++                    dpaths = manager.FindDeviceStringMatch('storage.originating_device', path)
++                    for dpath in dpaths:
++                        devif = dbus.Interface(bus.get_object('org.freedesktop.Hal', dpath), 'org.freedesktop.Hal.Device')
++                        try:
++                            vpaths = manager.FindDeviceStringMatch('block.storage_device', dpath)
++                            for vpath in vpaths:
++                                try:
++                                    vdevif = dbus.Interface(bus.get_object('org.freedesktop.Hal', vpath), 'org.freedesktop.Hal.Device')
++                                    if not vdevif.GetProperty('block.is_volume'):
++                                        continue
++                                    if vdevif.GetProperty('volume.fsusage') != 'filesystem':
++                                        continue
++                                    volif = dbus.Interface(bus.get_object('org.freedesktop.Hal', vpath), 'org.freedesktop.Hal.Device.Volume')
++                                    pdevif = dbus.Interface(bus.get_object('org.freedesktop.Hal', vdevif.GetProperty('info.parent')), 'org.freedesktop.Hal.Device')
++                                    vol = {'node': pdevif.GetProperty('block.device'),
++                                            'dev': vdevif,
++                                            'vol': volif,
++                                            'label': vdevif.GetProperty('volume.label')}
++                                    vols.append(vol)
++                                except dbus.exceptions.DBusException, e:
++                                    print e
++                                    continue
++                        except dbus.exceptions.DBusException, e:
++                            print e
++                            continue
++            except dbus.exceptions.DBusException, e:
++                continue
++
++        def ocmp(x,y):
++            if x['node'] < y['node']:
++                return -1
++            if x['node'] > y['node']:
++                return 1
++            return 0
++
++        vols.sort(cmp=ocmp)
++
++        if verbose:
++            print "FBSD:	", vols
++        
+         mtd=0
+-        label="READER"      # could use something more unique, like S/N or productID...
+-        cmd = '/usr/local/bin/calibre-mount-helper'
+-        cmd = [cmd, 'mount']
+-        for i in range(0,ndevs):
+-            cmd2="ls /dev/"+devs[i]+"*"
+-            p=subprocess.Popen(cmd2, shell=True, stdout=subprocess.PIPE)
+-            devs[i]=subprocess.Popen(["cut", "-d", "/", "-f" "3"], stdin=p.stdout, stdout=subprocess.PIPE).communicate()[0]
+-            p.stdout.close()
+-
+-            # try all the nodes to see what we can mount
+-            for dev in devs[i].split():
+-                mp='/media/'+label+'-'+dev
+-                mmp = mp
+-                if mmp.endswith('/'):
+-                    mmp = mmp[:-1]
+-                #print "trying ", dev, "on", mp
+-                try:
+-                    p = subprocess.Popen(cmd + ["/dev/"+dev, mmp])
+-                except OSError:
+-                    raise DeviceError(_('Could not find mount helper: %s.')%cmd[0])
+-                while p.poll() is None:
+-                    time.sleep(0.1)
+-
+-                if p.returncode == 0:
+-                    #print "  mounted", dev
+-                    if i == 0:
+-                        self._main_prefix = mp
+-                        self._main_dev = "/dev/"+dev
+-                        #print "main = ", self._main_dev, self._main_prefix
+-                    if i == 1:
+-                        self._card_a_prefix = mp
+-                        self._card_a_dev = "/dev/"+dev
+-                        #print "card a = ", self._card_a_dev, self._card_a_prefix
+-                    if i == 2:
+-                        self._card_b_prefix = mp
+-                        self._card_b_dev = "/dev/"+dev
+-                        #print "card b = ", self._card_b_dev, self._card_b_prefix
+ 
+-                    mtd += 1
+-                    break
++        for vol in vols:
++            mp = ''
++            if vol['dev'].GetProperty('volume.is_mounted'):
++                mp = vol['dev'].GetProperty('volume.mount_point')
++            else:
++                if verbose:
++                    print "FBSD:	trying ", vol['label'], "on", 'Calibre-'+labels[i]
++                try:
++                    vol['vol'].Mount('Calibre-'+vol['label'],
++                            vol['dev'].GetProperty('volume.fstype'), [])
++                    loops = 0
++                    while not vol['dev'].GetProperty('volume.is_mounted'):
++                        time.sleep(1)
++                        loops += 1
++                        if loops > 100:
++                            print "ERROR: Timeout waiting for mount to complete"
++                            continue
++                    mp = vol['dev'].GetProperty('volume.mount_point')
++                except dbus.exceptions.DBusException, e:
++                    print "Failed to mount ", e
++                    continue
++
++            # Mount Point becomes Mount Path
++            mp += '/'
++
++            if verbose:
++                print "FBSD:	  mounted", vol['label'], "on", mp
++            if mtd == 0:
++                self._main_prefix = mp
++                self._main_vol = vol['vol']
++                if verbose:
++                    print "FBSD:	main = ", self._main_prefix
++            if mtd == 1:
++                self._card_a_prefix = mp
++                self._card_a_vol = vol['vol']
++                if verbose:
++                    print "FBSD:	card a = ", self._card_a_prefix
++            if mtd == 2:
++                self._card_b_prefix = mp
++                self._card_b_vol = vol['vol']
++                if verbose:
++                    print "FBSD:	card b = ", self._card_b_prefix
++                # Note that mtd is used as a bool... not incrementing is fine.
++                break
++            mtd += 1
+ 
+         if mtd > 0:
+             return True
+-        else :
+-            return False
++        raise DeviceError(_('Unable to mount the device'))
++
+ #
+ # ------------------------------------------------------
+ #
+-#   this one is pretty simple:
+-#       just umount each of the previously
+-#       mounted filesystems, using the mount helper
++#    this one is pretty simple:
++#        just umount each of the previously
++#        mounted filesystems, using the stored volume object
+ #
+     def eject_freebsd(self):
+-        cmd = '/usr/local/bin/calibre-mount-helper'
+-        cmd = [cmd, 'eject']
++        import dbus
++        # There should be some way to access the -v arg...
++        verbose = False
+ 
+         if self._main_prefix:
+-            #print "umount main:", cmd, self._main_dev, self._main_prefix
++            if verbose:
++                print "FBSD:	umount main:", self._main_prefix
+             try:
+-                p = subprocess.Popen(cmd + [self._main_dev, self._main_prefix])
+-            except OSError:
+-                raise DeviceError(
+-                _('Could not find mount helper: %s.')%cmd[0])
+-            while p.poll() is None:
+-                time.sleep(0.1)
++                self._main_vol.Unmount([])
++            except dbus.exceptions.DBusException, e:
++                print 'Unable to eject ', e
+ 
+         if self._card_a_prefix:
+-            #print "umount card a:", cmd, self._card_a_dev, self._card_a_prefix
++            if verbose:
++                print "FBSD:	umount card a:", self._card_a_prefix
+             try:
+-                p = subprocess.Popen(cmd + [self._card_a_dev,  self._card_a_prefix])
+-            except OSError:
+-                raise DeviceError(
+-                _('Could not find mount helper: %s.')%cmd[0])
+-            while p.poll() is None:
+-                time.sleep(0.1)
++                self._card_a_vol.Unmount([])
++            except dbus.exceptions.DBusException, e:
++                print 'Unable to eject ', e
+ 
+         if self._card_b_prefix:
+-            #print "umount card b:", cmd, self._card_b_dev, self._card_b_prefix
++            if verbose:
++                print "FBSD:	umount card b:", self._card_b_prefix
+             try:
+-                p = subprocess.Popen(cmd + [self._card_b_dev, self._card_b_prefix])
+-            except OSError:
+-                raise DeviceError(
+-                _('Could not find mount helper: %s.')%cmd[0])
+-            while p.poll() is None:
+-                time.sleep(0.1)
++                self._card_b_vol.Unmount([])
++            except dbus.exceptions.DBusException, e:
++                print 'Unable to eject ', e
+ 
+         self._main_prefix = None
+         self._card_a_prefix = None
+@@ -859,11 +881,10 @@
+                     time.sleep(7)
+                     self.open_linux()
+             if isfreebsd:
+-                self._main_dev = self._card_a_dev = self._card_b_dev = None
++                self._main_vol = self._card_a_vol = self._card_b_vol = None
+                 try:
+                     self.open_freebsd()
+                 except DeviceError:
+-                    subprocess.Popen(["camcontrol", "rescan", "all"])
+                     time.sleep(2)
+                     self.open_freebsd()
+             if iswindows:
diff --git a/deskutils/calibre/pkg-message b/deskutils/calibre/pkg-message
new file mode 100644
index 000000000000..ee5420b75f18
--- /dev/null
+++ b/deskutils/calibre/pkg-message
@@ -0,0 +1,34 @@
+---
+
+NOTE: Calibre requires Hal to be running for access to devices.  The user who
+is running Calibre also needs to be able to mount devices.  For non-root users,
+this means modifying /usr/local/etc/PolicyKit/PolicyKit.conf and adding the
+policy there.
+
+To allow a single user named "example" to mount removable devices, add the
+following:
+
+  <match action="org.freedesktop.hal.storage.mount-removable">
+    <match user="example">
+      <return result="yes" />
+    </match>
+  </match> 
+
+Additional users will each need their own <match user="user"> block with a
+"yes" return result.
+
+To permit all accounts to mount removable devices (allowing a denial of access
+attack) add the following:
+
+  <match action="org.freedesktop.hal.storage.mount-removable">
+    <return result="yes" />
+  </match>
+
+---
+
+If you are using an Android device as your e-book reader, please note
+that the stock Google Books app uses a storage format that is not
+supported by calibre.  The calibre FAQ recommends that either of the
+free apps Aldiko or WordPlayer are used instead.
+
+---
diff --git a/deskutils/calibre/pkg-plist b/deskutils/calibre/pkg-plist
index 58d06e39d6dd..11c9606394fe 100644
--- a/deskutils/calibre/pkg-plist
+++ b/deskutils/calibre/pkg-plist
@@ -131,41 +131,7 @@ lib/calibre/calibre/ebooks/__init__.py
 lib/calibre/calibre/ebooks/azw4/__init__.py
 lib/calibre/calibre/ebooks/azw4/reader.py
 lib/calibre/calibre/ebooks/azw4/input.py
-lib/calibre/calibre/ebooks/chardet/__init__.py
-lib/calibre/calibre/ebooks/chardet/big5freq.py
-lib/calibre/calibre/ebooks/chardet/big5prober.py
-lib/calibre/calibre/ebooks/chardet/chardistribution.py
-lib/calibre/calibre/ebooks/chardet/charsetgroupprober.py
-lib/calibre/calibre/ebooks/chardet/charsetprober.py
-lib/calibre/calibre/ebooks/chardet/codingstatemachine.py
-lib/calibre/calibre/ebooks/chardet/constants.py
-lib/calibre/calibre/ebooks/chardet/escprober.py
-lib/calibre/calibre/ebooks/chardet/escsm.py
-lib/calibre/calibre/ebooks/chardet/eucjpprober.py
-lib/calibre/calibre/ebooks/chardet/euckrfreq.py
-lib/calibre/calibre/ebooks/chardet/euckrprober.py
-lib/calibre/calibre/ebooks/chardet/euctwfreq.py
-lib/calibre/calibre/ebooks/chardet/euctwprober.py
-lib/calibre/calibre/ebooks/chardet/gb2312freq.py
-lib/calibre/calibre/ebooks/chardet/gb2312prober.py
-lib/calibre/calibre/ebooks/chardet/hebrewprober.py
-lib/calibre/calibre/ebooks/chardet/jisfreq.py
-lib/calibre/calibre/ebooks/chardet/jpcntx.py
-lib/calibre/calibre/ebooks/chardet/langbulgarianmodel.py
-lib/calibre/calibre/ebooks/chardet/langcyrillicmodel.py
-lib/calibre/calibre/ebooks/chardet/langgreekmodel.py
-lib/calibre/calibre/ebooks/chardet/langhebrewmodel.py
-lib/calibre/calibre/ebooks/chardet/langhungarianmodel.py
-lib/calibre/calibre/ebooks/chardet/langthaimodel.py
-lib/calibre/calibre/ebooks/chardet/latin1prober.py
-lib/calibre/calibre/ebooks/chardet/mbcharsetprober.py
-lib/calibre/calibre/ebooks/chardet/mbcsgroupprober.py
-lib/calibre/calibre/ebooks/chardet/mbcssm.py
-lib/calibre/calibre/ebooks/chardet/sbcharsetprober.py
-lib/calibre/calibre/ebooks/chardet/sbcsgroupprober.py
-lib/calibre/calibre/ebooks/chardet/sjisprober.py
-lib/calibre/calibre/ebooks/chardet/universaldetector.py
-lib/calibre/calibre/ebooks/chardet/utf8prober.py
+lib/calibre/calibre/ebooks/chardet.py
 lib/calibre/calibre/ebooks/chm/__init__.py
 lib/calibre/calibre/ebooks/chm/input.py
 lib/calibre/calibre/ebooks/chm/metadata.py
@@ -333,11 +299,12 @@ lib/calibre/calibre/ebooks/odt/__init__.py
 lib/calibre/calibre/ebooks/odt/input.py
 lib/calibre/calibre/ebooks/oeb/__init__.py
 lib/calibre/calibre/ebooks/oeb/base.py
-lib/calibre/calibre/ebooks/oeb/entitydefs.py
 lib/calibre/calibre/ebooks/oeb/iterator.py
 lib/calibre/calibre/ebooks/oeb/output.py
+lib/calibre/calibre/ebooks/oeb/parse_utils.py
 lib/calibre/calibre/ebooks/oeb/reader.py
 lib/calibre/calibre/ebooks/oeb/stylizer.py
+lib/calibre/calibre/ebooks/oeb/display/test/test.py
 lib/calibre/calibre/ebooks/oeb/transforms/__init__.py
 lib/calibre/calibre/ebooks/oeb/transforms/cover.py
 lib/calibre/calibre/ebooks/oeb/transforms/filenames.py
@@ -686,6 +653,7 @@ lib/calibre/calibre/gui2/dialogs/tweak_epub_ui.py
 lib/calibre/calibre/gui2/dialogs/user_profiles.py
 lib/calibre/calibre/gui2/dialogs/user_profiles_ui.py
 lib/calibre/calibre/gui2/dnd.py
+lib/calibre/calibre/gui2/duplicates.py
 lib/calibre/calibre/gui2/ebook_download.py
 lib/calibre/calibre/gui2/email.py
 lib/calibre/calibre/gui2/filename_pattern_ui.py
@@ -775,11 +743,14 @@ lib/calibre/calibre/gui2/store/basic_config.py
 lib/calibre/calibre/gui2/store/opensearch_store.py
 lib/calibre/calibre/gui2/store/web_store_dialog.py
 lib/calibre/calibre/gui2/store/basic_config_widget_ui.py
+lib/calibre/calibre/gui2/store/stores/amazon_es_plugin.py
 lib/calibre/calibre/gui2/store/stores/amazon_fr_plugin.py
+lib/calibre/calibre/gui2/store/stores/amazon_it_plugin.py
 lib/calibre/calibre/gui2/store/stores/beam_ebooks_de_plugin.py
 lib/calibre/calibre/gui2/store/stores/ebookpoint_plugin.py
 lib/calibre/calibre/gui2/store/stores/escapemagazine_plugin.py
 lib/calibre/calibre/gui2/store/stores/gandalf_plugin.py
+lib/calibre/calibre/gui2/store/stores/litres_plugin.py
 lib/calibre/calibre/gui2/store/stores/pragmatic_bookshelf_plugin.py
 lib/calibre/calibre/gui2/store/stores/nexto_plugin.py
 lib/calibre/calibre/gui2/store/stores/rw2010_plugin.py
@@ -1009,6 +980,9 @@ lib/calibre/calibre/utils/sigil.py
 lib/calibre/calibre/utils/smartypants.py
 lib/calibre/calibre/utils/smtp.py
 lib/calibre/calibre/utils/smtplib.py
+lib/calibre/calibre/utils/soupparser.py
+lib/calibre/calibre/utils/coffeescript.py
+lib/calibre/calibre/utils/linux_trash.py
 lib/calibre/calibre/utils/terminfo.py
 lib/calibre/calibre/utils/text2int.py
 lib/calibre/calibre/utils/threadpool.py
@@ -1035,14 +1009,52 @@ lib/calibre/calibre/web/jsbrowser/__init__.py
 lib/calibre/calibre/web/jsbrowser/login.py
 lib/calibre/calibre/web/jsbrowser/test.py
 lib/calibre/calibre/web/jsbrowser/forms.py
+lib/calibre/chardet/euctwprober.py
+lib/calibre/chardet/hebrewprober.py
+lib/calibre/chardet/euckrfreq.py
+lib/calibre/chardet/charsetgroupprober.py
+lib/calibre/chardet/latin1prober.py
+lib/calibre/chardet/mbcssm.py
+lib/calibre/chardet/langthaimodel.py
+lib/calibre/chardet/euckrprober.py
+lib/calibre/chardet/escprober.py
+lib/calibre/chardet/mbcharsetprober.py
+lib/calibre/chardet/langcyrillicmodel.py
+lib/calibre/chardet/__init__.py
+lib/calibre/chardet/chardistribution.py
+lib/calibre/chardet/sbcharsetprober.py
+lib/calibre/chardet/sjisprober.py
+lib/calibre/chardet/langbulgarianmodel.py
+lib/calibre/chardet/langhebrewmodel.py
+lib/calibre/chardet/gb2312prober.py
+lib/calibre/chardet/escsm.py
+lib/calibre/chardet/utf8prober.py
+lib/calibre/chardet/codingstatemachine.py
+lib/calibre/chardet/test.py
+lib/calibre/chardet/big5freq.py
+lib/calibre/chardet/gb2312freq.py
+lib/calibre/chardet/langgreekmodel.py
+lib/calibre/chardet/euctwfreq.py
+lib/calibre/chardet/eucjpprober.py
+lib/calibre/chardet/jpcntx.py
+lib/calibre/chardet/jisfreq.py
+lib/calibre/chardet/charsetprober.py
+lib/calibre/chardet/sbcsgroupprober.py
+lib/calibre/chardet/constants.py
+lib/calibre/chardet/mbcsgroupprober.py
+lib/calibre/chardet/langhungarianmodel.py
+lib/calibre/chardet/universaldetector.py
+lib/calibre/chardet/big5prober.py
 lib/calibre/cherrypy/__init__.py
-lib/calibre/cherrypy/_cpcgifs.py
 lib/calibre/cherrypy/_cpchecker.py
+lib/calibre/cherrypy/_cpcompat.py
 lib/calibre/cherrypy/_cpconfig.py
 lib/calibre/cherrypy/_cpdispatch.py
 lib/calibre/cherrypy/_cperror.py
 lib/calibre/cherrypy/_cplogging.py
 lib/calibre/cherrypy/_cpmodpy.py
+lib/calibre/cherrypy/_cpnative_server.py
+lib/calibre/cherrypy/_cpreqbody.py
 lib/calibre/cherrypy/_cprequest.py
 lib/calibre/cherrypy/_cpserver.py
 lib/calibre/cherrypy/_cpthreadinglocal.py
@@ -1052,19 +1064,23 @@ lib/calibre/cherrypy/_cpwsgi.py
 lib/calibre/cherrypy/_cpwsgi_server.py
 lib/calibre/cherrypy/lib/__init__.py
 lib/calibre/cherrypy/lib/auth.py
+lib/calibre/cherrypy/lib/auth_basic.py
+lib/calibre/cherrypy/lib/auth_digest.py
 lib/calibre/cherrypy/lib/caching.py
 lib/calibre/cherrypy/lib/covercp.py
+lib/calibre/cherrypy/lib/cpstats.py
 lib/calibre/cherrypy/lib/cptools.py
 lib/calibre/cherrypy/lib/encoding.py
+lib/calibre/cherrypy/lib/gctools.py
 lib/calibre/cherrypy/lib/http.py
 lib/calibre/cherrypy/lib/httpauth.py
+lib/calibre/cherrypy/lib/httputil.py
+lib/calibre/cherrypy/lib/jsontools.py
 lib/calibre/cherrypy/lib/profiler.py
-lib/calibre/cherrypy/lib/safemime.py
+lib/calibre/cherrypy/lib/reprconf.py
 lib/calibre/cherrypy/lib/sessions.py
 lib/calibre/cherrypy/lib/static.py
-lib/calibre/cherrypy/lib/tidy.py
-lib/calibre/cherrypy/lib/wsgiapp.py
-lib/calibre/cherrypy/lib/xmlrpc.py
+lib/calibre/cherrypy/lib/xmlrpcutil.py
 lib/calibre/cherrypy/process/__init__.py
 lib/calibre/cherrypy/process/plugins.py
 lib/calibre/cherrypy/process/servers.py
@@ -1072,6 +1088,46 @@ lib/calibre/cherrypy/process/win32.py
 lib/calibre/cherrypy/process/wspbus.py
 lib/calibre/cherrypy/scaffold/__init__.py
 lib/calibre/cherrypy/wsgiserver/__init__.py
+lib/calibre/cherrypy/wsgiserver/ssl_builtin.py
+lib/calibre/cherrypy/wsgiserver/ssl_pyopenssl.py
+lib/calibre/cherrypy/wsgiserver/wsgiserver2.py
+lib/calibre/cherrypy/wsgiserver/wsgiserver3.py
+lib/calibre/html5lib/sanitizer.py
+lib/calibre/html5lib/tokenizer.py
+lib/calibre/html5lib/html5parser.py
+lib/calibre/html5lib/ihatexml.py
+lib/calibre/html5lib/__init__.py
+lib/calibre/html5lib/tokenizer_old.py
+lib/calibre/html5lib/inputstream.py
+lib/calibre/html5lib/utils.py
+lib/calibre/html5lib/constants.py
+lib/calibre/html5lib/filters/lint.py
+lib/calibre/html5lib/filters/sanitizer.py
+lib/calibre/html5lib/filters/whitespace.py
+lib/calibre/html5lib/filters/__init__.py
+lib/calibre/html5lib/filters/inject_meta_charset.py
+lib/calibre/html5lib/filters/_base.py
+lib/calibre/html5lib/filters/formfiller.py
+lib/calibre/html5lib/filters/optionaltags.py
+lib/calibre/html5lib/serializer/xhtmlserializer.py
+lib/calibre/html5lib/serializer/__init__.py
+lib/calibre/html5lib/serializer/htmlserializer.py
+lib/calibre/html5lib/treewalkers/dom.py
+lib/calibre/html5lib/treewalkers/genshistream.py
+lib/calibre/html5lib/treewalkers/etree.py
+lib/calibre/html5lib/treewalkers/__init__.py
+lib/calibre/html5lib/treewalkers/pulldom.py
+lib/calibre/html5lib/treewalkers/soup.py
+lib/calibre/html5lib/treewalkers/lxmletree.py
+lib/calibre/html5lib/treewalkers/_base.py
+lib/calibre/html5lib/treewalkers/simpletree.py
+lib/calibre/html5lib/treebuilders/dom.py
+lib/calibre/html5lib/treebuilders/etree.py
+lib/calibre/html5lib/treebuilders/__init__.py
+lib/calibre/html5lib/treebuilders/etree_lxml.py
+lib/calibre/html5lib/treebuilders/soup.py
+lib/calibre/html5lib/treebuilders/_base.py
+lib/calibre/html5lib/treebuilders/simpletree.py
 lib/calibre/odf/__init__.py
 lib/calibre/odf/anim.py
 lib/calibre/odf/attrconverters.py
@@ -1191,6 +1247,7 @@ share/calibre/content_server/star-half.png
 share/calibre/content_server/star-off.png
 share/calibre/content_server/star-on.png
 share/calibre/default_tweaks.py
+share/calibre/display/cfi.js
 share/calibre/ebook-convert-complete.pickle
 share/calibre/fonts/liberation/LiberationMono-Bold.ttf
 share/calibre/fonts/liberation/LiberationMono-BoldItalic.ttf
@@ -1454,6 +1511,7 @@ share/calibre/viewer/referencing.js
 @dirrm share/calibre/fonts/prs500
 @dirrm share/calibre/fonts/liberation
 @dirrm share/calibre/fonts
+@dirrm share/calibre/display
 @dirrm share/calibre/content_server/read
 @dirrm share/calibre/content_server/jquery_ui/js
 @dirrm share/calibre/content_server/jquery_ui/css/humanity-custom/images
@@ -1468,6 +1526,7 @@ share/calibre/viewer/referencing.js
 @dirrm lib/calibre/routes
 @dirrm lib/calibre/pyPdf
 @dirrm lib/calibre/odf
+@dirrm lib/calibre/chardet
 @dirrm lib/calibre/cherrypy/wsgiserver
 @dirrm lib/calibre/cherrypy/scaffold
 @dirrm lib/calibre/cherrypy/process
@@ -1537,6 +1596,8 @@ share/calibre/viewer/referencing.js
 @dirrm lib/calibre/calibre/ebooks/pdb/palmdoc
 @dirrm lib/calibre/calibre/ebooks/pdb/ereader
 @dirrm lib/calibre/calibre/ebooks/pdb
+@dirrm lib/calibre/calibre/ebooks/oeb/display/test
+@dirrm lib/calibre/calibre/ebooks/oeb/display
 @dirrm lib/calibre/calibre/ebooks/oeb/transforms
 @dirrm lib/calibre/calibre/ebooks/oeb
 @dirrm lib/calibre/calibre/ebooks/odt
@@ -1565,7 +1626,6 @@ share/calibre/viewer/referencing.js
 @dirrm lib/calibre/calibre/ebooks/compression
 @dirrm lib/calibre/calibre/ebooks/comic
 @dirrm lib/calibre/calibre/ebooks/chm
-@dirrm lib/calibre/calibre/ebooks/chardet
 @dirrm lib/calibre/calibre/ebooks/azw4
 @dirrm lib/calibre/calibre/ebooks
 @dirrm lib/calibre/calibre/db/tests
@@ -1604,6 +1664,11 @@ share/calibre/viewer/referencing.js
 @dirrm lib/calibre/calibre/devices
 @dirrm lib/calibre/calibre/customize
 @dirrm lib/calibre/calibre
+@dirrm lib/calibre/html5lib/filters
+@dirrm lib/calibre/html5lib/serializer
+@dirrm lib/calibre/html5lib/treewalkers
+@dirrm lib/calibre/html5lib/treebuilders
+@dirrm lib/calibre/html5lib
 @dirrm lib/calibre
 @dirrmtry %%PYTHON_SITELIBDIR%%
 @dirrmtry %%PYTHON_LIBDIR%%