Openpyxl engine for reading excel files

pandas/io/excel/_base.py

@@ -400,7 +400,7 @@ def parse(self,
             data = self.get_sheet_data(sheet, convert_float)
             usecols = _maybe_convert_usecols(usecols)
 
-            if sheet.nrows == 0:
+            if not data:
                 output[asheetname] = DataFrame()
                 continue
 

pandas/io/excel/_openpyxl.py

@@ -1,5 +1,16 @@
-from pandas.io.excel._base import ExcelWriter
+from collections import OrderedDict
+from io import BytesIO
+
+import pandas.compat as compat
+from pandas.core.dtypes.common import is_integer, is_list_like
+from pandas.core.frame import DataFrame
+from pandas.io.common import (_is_url, _urlopen, _validate_header_arg,
+                              get_filepath_or_buffer)
+from pandas.io.excel._base import (ExcelFile, ExcelWriter, _BaseExcelReader,
+                                   _fill_mi_header, _maybe_convert_to_string,
+                                   _maybe_convert_usecols, _pop_header_name)
 from pandas.io.excel._util import _validate_freeze_panes
+from pandas.io.parsers import _validate_usecols_arg, _validate_usecols_names
 
 
 class _OpenpyxlWriter(ExcelWriter):
@@ -451,3 +462,256 @@ def write_cells(self, cells, sheet_name=None, startrow=0, startcol=0,
                             xcell = wks.cell(column=col, row=row)
                             for k, v in style_kwargs.items():
                                 setattr(xcell, k, v)
+
+
+class _OpenpyxlReader(_BaseExcelReader):
+
+    def __init__(self, filepath_or_buffer):
+        """Reader using openpyxl engine.
+
+        Parameters
+        ----------
+        filepath_or_buffer : string, path object or Workbook
+            Object to be parsed.
+        """
+        err_msg = "Install xlrd >= 1.0.0 for Excel support"
+
+        try:
+            import openpyxl
+        except ImportError:
+            raise ImportError(err_msg)
+
+        # If filepath_or_buffer is a url, want to keep the data as bytes so
+        # can't pass to get_filepath_or_buffer()
+        if _is_url(filepath_or_buffer):
+            filepath_or_buffer = BytesIO(_urlopen(filepath_or_buffer).read())
+        elif not isinstance(filepath_or_buffer,
+                            (ExcelFile, openpyxl.Workbook)):
+            filepath_or_buffer, _, _, _ = get_filepath_or_buffer(
+                filepath_or_buffer)
+
+        if isinstance(filepath_or_buffer, openpyxl.Workbook):
+            self.book = filepath_or_buffer
+        elif hasattr(filepath_or_buffer, "read"):
+            if hasattr(filepath_or_buffer, 'seek'):
+                filepath_or_buffer.seek(0)
+            self.book = openpyxl.load_workbook(
+                filepath_or_buffer, data_only=True)
+        elif isinstance(filepath_or_buffer, compat.string_types):
+            self.book = openpyxl.load_workbook(
+                filepath_or_buffer, data_only=True)
+        else:
+            raise ValueError('Must explicitly set engine if not passing in'
+                             ' buffer or path for io.')
+
+    @property
+    def sheet_names(self):
+        return self.book.sheetnames
+
+    def get_sheet_by_name(self, name):
+        return self.book[name]
+
+    def get_sheet_by_index(self, index):
+        return self.book.worksheets[index]
+
+    @staticmethod
+    def _replace_type_error_with_nan(rows):
+        nan = float('nan')
+        for row in rows:
+            yield [nan
+                   if cell.data_type == cell.TYPE_ERROR
+                   else cell.value
+                   for cell in row]
+
+    def get_sheet_data(self, sheet, convert_float):
+        data = self._replace_type_error_with_nan(sheet.rows)
+        # TODO: support using iterator
+        # TODO: don't make strings out of data
+        return list(data)
+
+    def parse(self,
+              sheet_name=0,
+              header=0,
+              names=None,
+              index_col=None,
+              usecols=None,
+              squeeze=False,
+              converters=None,
+              dtype=None,
+              true_values=None,
+              false_values=None,
+              skiprows=None,
+              nrows=None,
+              na_values=None,
+              verbose=False,
+              parse_dates=False,
+              date_parser=None,
+              thousands=None,
+              comment=None,
+              skipfooter=0,
+              convert_float=True,
+              mangle_dupe_cols=True,
+              **kwds):
+
+        _validate_header_arg(header)
+
+        ret_dict = False
+
+        # Keep sheetname to maintain backwards compatibility.
+        if isinstance(sheet_name, list):
+            sheets = sheet_name
+            ret_dict = True
+        elif sheet_name is None:
+            sheets = self.sheet_names
+            ret_dict = True
+        else:
+            sheets = [sheet_name]
+
+        # handle same-type duplicates.
+        sheets = list(OrderedDict.fromkeys(sheets).keys())
+
+        output = OrderedDict()
+
+        for asheetname in sheets:
+            if verbose:
+                print("Reading sheet {sheet}".format(sheet=asheetname))
+
+            if isinstance(asheetname, compat.string_types):
+                sheet = self.get_sheet_by_name(asheetname)
+            else:  # assume an integer if not a string
+                sheet = self.get_sheet_by_index(asheetname)
+
+            data = self.get_sheet_data(sheet, convert_float)
+            if not data or data == [[None]]:
+                output[asheetname] = DataFrame()
+                continue
+
+            usecols = _maybe_convert_usecols(usecols)
+
+            if is_list_like(header) and len(header) == 1:
+                header = header[0]
+
+            # TODO: scrutinize what is going here
+            # forward fill and pull out names for MultiIndex column
+            header_names = None
+            if header is not None and is_list_like(header):
+                header_names = []
+                control_row = [True] * len(data[0])
+
+                for row in header:
+                    if is_integer(skiprows):
+                        row += skiprows
+
+                    data[row], control_row = _fill_mi_header(data[row],
+                                                             control_row)
+
+                    if index_col is not None:
+                        header_name, _ = _pop_header_name(data[row], index_col)
+                        header_names.append(header_name)
+
+            # TODO: implement whatever this should do
+            # has_index_names = is_list_like(header) and len(header) > 1
+
+            if skiprows:
+                data = [row for i, row in enumerate(data) if i not in skiprows]
+
+            if skipfooter:
+                data = data[:-skipfooter]
+
+            column_names = [cell for i, cell in enumerate(data.pop(0))]
+
+            frame = DataFrame(data, columns=column_names)
+            if usecols:
+                _validate_usecols_arg(usecols)
+                usecols = sorted(usecols)
+                if any(isinstance(i, str) for i in usecols):
+                    _validate_usecols_names(usecols, column_names)
+                    frame = frame[usecols]
+                else:
+                    frame = frame.iloc[:, usecols]
+
+            if not converters:
+                converters = dict()
+            if not dtype:
+                dtype = dict()
+
+            # handle columns referenced by number so all references are by
+            #  column name
+            handled_converters = {}
+            for k, v in converters.items():
+                if k not in frame.columns and isinstance(k, int):
+                    k = frame.columns[k]
+                handled_converters[k] = v
+            converters = handled_converters
+
+            # attempt to convert object columns to integer. Only because this
+            # is implicitly done when reading and excel file with xlrd
+            # TODO: question if this should be default behaviour
+            if len(frame) > 0:
+                for column in set(frame) - set(dtype.keys()):
+                    if frame[column].dtype == object:
+                        try:
+                            frame[column] = frame[column].astype('int64')
+                        except (ValueError, TypeError):
+                            try:
+                                frame[column] = frame[column].astype('float64')
+                            except (ValueError, TypeError):
+                                continue
+                    elif (convert_float and
+                            frame[column].dtype == float and
+                            all(frame[column] % 1 == 0)):
+                        frame[column] = frame[column].astype('int64')
+                    elif not convert_float:
+                        if frame[column].dtype == int:
+                            frame[column] = frame[column].astype('float64')
+
+            if converters:
+                for k, v in converters.items():
+                    # for compatibiliy reasons
+                    if frame[k].dtype == float and convert_float:
+                        frame[k] = frame[k].fillna('')
+                    frame[k] = frame[k].apply(v)
+
+            if dtype:
+                for k, v in dtype.items():
+                    frame[k] = frame[k].astype(v)
+
+            if index_col is not None:
+                if is_list_like(index_col):
+                    if any(isinstance(i, str) for i in index_col):
+                        # TODO: see if there is already a method for this in
+                        # pandas.io.parsers
+                        frame = frame.set_index(index_col)
+                        if len(index_col) == 1:
+                            # TODO: understand why this is needed
+                            raise TypeError(
+                                "list indices must be integers.*, not str")
+                    else:
+                        frame = frame.set_index(
+                            [column_names[i] for i in index_col])
+                else:
+                    if isinstance(index_col, str):
+                        frame = frame.set_index(index_col)
+                    else:
+                        frame = frame.set_index(column_names[index_col])
+
+            output[asheetname] = frame
+            if not squeeze or isinstance(output[asheetname], DataFrame):
+                if header_names:
+                    output[asheetname].columns = output[
+                        asheetname].columns.set_names(header_names)
+                elif compat.PY2:
+                    output[asheetname].columns = _maybe_convert_to_string(
+                        output[asheetname].columns)
+
+            # name unnamed columns
+            unnamed = 0
+            for i, col_name in enumerate(frame.columns.values):
+                if col_name is None:
+                    frame.columns.values[i] = "Unnamed: {n}".format(n=unnamed)
+                    unnamed += 1
+
+        if ret_dict:
+            return output
+        else:
+            return output[asheetname]