API documentation

class Field(object):
    '''
    Fields are the elements of information that you wish to extract from each document.

    Parameters:
        name:  a short hand name (name), which will be used as its key in the document
        extractor: an Extractor object that defines how this field's data can be
            extracted from source documents.
        required: whether this field is required. The `Reader` class should skip the
            document is the value for this Field is `None`.
        skip: if `True`, this field will not be included in the results.
    '''

    def __init__(self,
                 name: str,
                 extractor: extract.Extractor = extract.Constant(None),
                 required: bool = False,
                 skip: bool = False,
                 **kwargs
                 ):

        self.name = name
        self.extractor = extractor
        self.required = required
        self.skip = skip

class Reader:
    '''
    A base class for readers. Readers are objects that can generate documents
    from a source dataset.

    Subclasses of `Reader` can be created to read specific data formats. 
    In practice, you will probably work with a subclass of `Reader` like `XMLReader`,
    `CSVReader`, etc., that provides the core functionality for a file type, and create
    a subclass for a specific dataset.

    Some methods of this class need to be implemented in child classes, and will raise
    `NotImplementedError` if you try to use `Reader` directly.

    A fully implemented `Reader` subclass will define how to read a dataset by
    describing:

    - How to obtain its source files.
    - How to parse and iterate over source files.
    - What fields each document contains, and how to extract them from the source data.

    This requires implementing the following attributes/methods:

    - `fields`: a list of `Field` instances that describe the fields that will appear in
        documents, and how to extract their value.
    - `sources`: a method that returns an iterable of sources (e.g. file paths), possibly
        with metadata for each.
    - `data_directory` (optional): a string with the path to the directory containing
        the source data. You can use this in the implementation of `sources`; it's not
        used elsewhere.
    - `data_from_file` `data_from_bytes`, `data_from_response`: methods that respectively
        receive a file path, a byte sequence, or an HTTP response, and return a data
        object. (The type of the data will depend on how you implement your reader; this
        could be a parsed graph, a row iterator, etc.). You must implement at least one of
        these methods to have a functioning reader.
    - `iterate_data`: method that takes a data object (the output of
        `data_from_file`/`data_from_bytes`/`data_from_response`) and a metadata dictionary,
        iterates over the source data, and returns the data that should be passed on to
        extractors for each document.
    - `validate` (optional): a method that will check the reader configuration. This is
        useful for abstract readers like the `XMLReader`, `CSVReader`, etc., so they
        can verify a child class is implementing attributes correctly.

    Abstract reader types like `CSVReader` usually leave `fields` and `sources`
    unimplemented.
    '''

    @property
    def data_directory(self) -> str:
        '''
        Path to source data directory.

        Raises:
            NotImplementedError: This method needs to be implementd on child
                classes. It will raise an error by default.
        '''
        raise NotImplementedError('Reader missing data_directory')


    @property
    def fields(self) -> List[Field]:
        '''
        The list of fields that are extracted from documents.

        These should be instances of the `Field` class (or implement the same API).

        Raises:
            NotImplementedError: This method needs to be implementd on child
                classes. It will raise an error by default.
        '''
        raise NotImplementedError('Reader missing fields implementation')

    @property
    def fieldnames(self) -> List[str]:
        '''
        A list containing the name of each field of this Reader
        '''
        return [field.name for field in self.fields]


    @property
    def _required_field_names(self) -> List[str]:
        '''
        A list of the names of all required fields
        '''
        return [field.name for field in self.fields if field.required]


    def sources(self, **kwargs) -> Iterable[Source]:
        '''
        Obtain source files for the Reader.

        Returns:
            an iterable of tuples that each contain a string path, and a dictionary
                with associated metadata. The metadata can contain any data that was
                extracted before reading the file itself, such as data based on the
                file path, or on a metadata file.

        Raises:
            NotImplementedError: This method needs to be implementd on child
                classes. It will raise an error by default.
        '''
        raise NotImplementedError('Reader missing sources implementation')

    def source2dicts(self, source: Source, source_index=-1) -> Iterable[Document]:
        '''
        Given a source file, returns an iterable of extracted documents.

        Parameters:
            source: Source to extract.

        Returns:
            an iterable of document dictionaries. Each of these is a dictionary,
                where the keys are names of this Reader's `fields`, and the values
                are based on the extractor of each field.
        '''

        self.validate()

        data, metadata = self.data_and_metadata_from_source(source)

        if isinstance(data, AbstractContextManager):
            context_manager = data
        else:
            context_manager = nullcontext(data)

        with context_manager as data:
            for index, extracted_data in enumerate(self.iterate_data(data, metadata)):
                base_data = {
                    'metadata': metadata,
                    'index': index,
                    'source_index': source_index,
                }
                document_data = base_data | extracted_data
                document = self.extract_document(**document_data)
                if self._has_required_fields(document):
                    yield document


    def data_and_metadata_from_source(self, source: Source) -> Tuple[Any, Dict]:
        '''
        Extract the data and metadata object from a source.

        Parameters:
            source: Source to extract.

        Returns:
            A tuple with the parsed source data, and the metadata (empty if none was
                provided).
        '''
        if isinstance(source, tuple) and len(source) == 2:
            source_data, metadata = source
        else:
            source_data = source
            metadata = {}

        if isinstance(source_data, str):
            if not isfile(source_data):
                raise FileNotFoundError(f'Invalid file path: {source_data}')
            data = self.data_from_file(source_data)
        elif isinstance(source_data, bytes):
            data = self.data_from_bytes(source_data)
        elif isinstance(source_data, Response):
            data = self.data_from_response(source_data)
        else:
            raise TypeError(f'Unknown source type: {type(source_data)}')

        return data, metadata


    def data_from_file(self, path: str) -> Any:
        '''
        Extract source data from a filename.

        The return type depends on how the reader is implemented, but should be some kind
        of data structure from which documents can be extracted. It serves as the input
        to `self.iterate_data`.

        This method can also return a context manager. This is especially useful to
        iterate over large files in `iterate_data`, without loading the complete file
        contents in memory.

        Tip: if you have implemented `self.data_from_bytes`, this method can probably just
        read the binary contents of the file and call that method.

        Parameters:
            path: The path to a file.

        Returns:
            A data object. The type depends on the reader implementation.

        Raises:
            NotImplementedError: this method may be implemented on child classes, but
                has no default implementation.
        '''

        raise NotImplementedError('This reader does not support filename input')


    def data_from_bytes(self, bytes: bytes) -> Any:
        '''
        Extract source data from a bytes object. Like `data_from_file`, but with bytes
        input.

        Parameters:
            bytes: byte contents of the source

        Returns:
            A data object. The type depends on the reader implementation. This may also
                be a context manager.

        Raises:
            NotImplementedError: this method may be implemented on child classes, but
                has no default implementation.
        '''

        raise NotImplementedError('This reader does not support bytes input')


    def data_from_response(self, response: Response) -> Any:
        '''
        Extract data from an HTTP response. Like `data_from_file`, but with `Response`
        input.

        Parameters:
            response: HTTP response object

        Returns:
            A data object. The type depends on the reader implementation. This may also
                be a context manager.

        Raises:
            NotImplementedError: this method may be implemented on child classes, but has
                no default implementation.
        '''
        raise NotImplementedError('This reader does not support Response input')


    def iterate_data(self, data: Any, metadata: Dict) -> Iterable[Document]:
        '''
        Iterate parsed source data, return data for each document.

        This should return the arguments that are passed on to field extractors per
        document. These usually cater to a specific extractor type. For example, the
        `CSVReader` returns an argument `rows`, which is used by the `CSV` extractor.

        The core `source2dicts` method will also provide `metadata` and `index` arguments
        to extractors, which you may override by providing them here.

        Parameters:
            data: The data object from a source. The type depends on the reader
                implementation; this is the output of `self.data_from_file` or
                `self.data_from_bytes`.
            metadata: Dictionary containing metadata for the source.

        Returns:
            An iterable of dictionaries. Each iteration will be extracted as a single
            document. The items in the dictionary are given as arguments to field
            extractors.

        Raises:
            NotImplementedError: This method must be implemented on child classes. It
                will raise an error otherwise.
        '''
        raise NotImplementedError('Data iteration is not implemented')


    def extract_document(
            self,
            **kwargs
        ) -> Document:
        '''
        Extract each field of a document, based on the raw data for the document
        '''
        return {
            field.name: field.extractor.apply(**kwargs)
            for field in self.fields
            if not field.skip
        }

    def documents(self, sources:Iterable[Source] = None) -> Iterable[Document]:
        '''
        Returns an iterable of extracted documents from source files.

        Parameters:
            sources: an iterable of paths to source files. If omitted, the reader
                class will use the value of `self.sources()` instead.

        Returns:
            an iterable of document dictionaries. Each of these is a dictionary,
                where the keys are names of this Reader's `fields`, and the values
                are based on the extractor of each field.
        '''
        sources = sources or self.sources()
        return (
            document
            for i, source in enumerate(sources)
            for document in self.source2dicts(
                source, source_index=i
            )
        )

    def export_csv(self, path: str, sources: Optional[Iterable[Source]] = None) -> None:
        '''
        Extracts documents from sources and saves them in a CSV file.

        This will write a CSV file in the provided `path`. This method has no return
        value.

        Parameters:
            path: the path where the CSV file should be saved.
            sources: an iterable of paths to source files. If omitted, the reader class
                will use the value of `self.sources()` instead.
        '''
        documents = self.documents(sources)

        with open(path, 'w') as outfile:
            writer = csv.DictWriter(outfile, self.fieldnames)
            writer.writeheader()
            for doc in documents:
                writer.writerow(doc)


    def validate(self):
        '''
        Validate that the reader is configured properly.

        This is a good place to check parameters that are overridden in a child class. A
        common use case is use self._reject_extractors to raise an error if any fields use
        unsupported extractor types.
        '''
        pass

    def _reject_extractors(self, *inapplicable_extractors: extract.Extractor):
        '''
        Raise errors if any fields use any of the given extractors.

        This can be used to check that fields use extractors that match
        the Reader subclass.

        Raises:
            RuntimeError: raised when a field uses an extractor that is provided
                in the input.
        '''
        for field in self.fields:
            if isinstance(field.extractor, inapplicable_extractors):
                raise RuntimeError(
                    "Specified extractor method cannot be used with this type of data")

    def _has_required_fields(self, document: Document) -> Iterable[Document]:
        '''
        Check whether a document has a value for all fields marked as required.
        '''

        has_field = lambda field_name: document.get(field_name, None) is not None
        return all(
            has_field(field_name) for field_name in self._required_field_names
        )

class CSVReader(Reader):
    '''
    A base class for Readers of .csv (comma separated value) files.

    The CSVReader is designed for .csv or .tsv files that have a header row, and where
    each file may list multiple documents.

    The data should be structured in one of the following ways:

    - one document per row (this is the default)
    - each document spans a number of consecutive rows. In this case, there should be a
        column that indicates the identity of the document.

    In addition to generic extractor classes, this reader supports the `CSV` extractor.
    '''

    field_entry = None
    '''
    If applicable, the name of the column that identifies entries. Subsequent rows with the
    same value for this column are treated as a single document. If left blank, each row
    is treated as a document.
    '''

    required_field = None
    '''
    Specifies the name of a required column in the CSV data, for example the main content.
    Rows with an empty value for `required_field` will be skipped.
    '''

    delimiter = ','
    '''
    The column delimiter used in the CSV data
    '''

    skip_lines = 0
    '''
    Number of lines in the file to skip before reading the header. Can be used when files
    use a fixed "preamble", e.g. to describe metadata or provenance.
    '''


    def validate(self):
        # make sure the field size is as big as the system permits
        csv.field_size_limit(sys.maxsize)
        self._reject_extractors(extract.XML)


    @contextmanager
    def data_from_file(self, path: str):
        with open(path, 'r') as f:
            logger.info('Reading CSV file {}...'.format(path))

            # skip first n lines
            for _ in range(self.skip_lines):
                next(f)

            reader = csv.DictReader(f, delimiter=self.delimiter)
            yield reader


    def iterate_data(self, data: csv.DictReader, metadata) -> Iterable[Document]:
        document_id = None
        rows = []
        for row in data:
            is_new_document = True

            if self.required_field and not row.get(self.required_field):  # skip row if required_field is empty
                continue

            if self.field_entry:
                identifier = row[self.field_entry]
                if identifier == document_id:
                    is_new_document = False
                else:
                    document_id = identifier

            if is_new_document and rows:
                yield {'rows': rows, 'metadata': metadata}
                rows = [row]
            else:
                rows.append(row)

        yield {'rows': rows}

class XLSXReader(Reader):
    '''
    A base class for Readers that extract data from .xlsx spreadsheets

    The XLSXReader is quite rudimentary, and is designed to extract data from
    spreadsheets that are formatted like a CSV table, with a clear column layout. The
    sheet should have a header row.

    The data should be structured in one of the following ways:

    - one document per row (this is the default)
    - each document spans a number of consecutive rows. In this case, there should be a
        column that indicates the identity of the document.

    The XLSXReader will only look at the _first_ sheet in each file.

    In addition to generic extractor classes, this reader supports the `CSV` extractor.
    '''

    field_entry = None
    '''
    If applicable, the name of column that identifies entries. Subsequent rows with the
    same value for this column are treated as a single document. If left blank, each row
    is treated as a document.
    '''

    required_field = None
    '''
    Specifies the name of a required column, for example the main content. Rows with
    an empty value for `required_field` will be skipped.
    '''

    skip_lines = 0
    '''
    Number of lines in the sheet to skip before reading the header. Can be used when files
    use a fixed "preamble", e.g. to describe metadata or provenance.
    '''


    def validate(self):
        self._reject_extractors(extract.XML)


    def data_from_file(self, path) -> Workbook:
        logger.info('Reading XLSX file {}...'.format(path))
        return openpyxl.load_workbook(path)


    def iterate_data(self, data: Workbook, metadata: Dict):
        sheets = data.sheetnames
        sheet = data[sheets[0]]
        return self._sheet2dicts(sheet, metadata)


    def _sheet2dicts(self, sheet: Worksheet, metadata):
        '''
        Extract documents from a single worksheet
        '''

        data = (row for row in sheet.values)

        for _ in range(self.skip_lines):
            next(data)

        header = list(next(data))

        document_id = None
        rows = []

        for row in data:
            values = {
                col: value
                for col, value in zip(header, row)
            }

            # skip row if required_field is empty
            if self.required_field and not values.get(self.required_field):
                continue

            identifier = values.get(self.field_entry, None)
            is_new_document = identifier == None or identifier != document_id
            document_id = identifier

            if is_new_document and rows:
                yield {'rows': rows}
                rows = [values]
            else:
                rows.append(values)

        if rows:
            yield {'rows': rows}

class XMLReader(Reader):
    '''
    A base class for Readers that extract data from XML files.

    The built-in functionality of the XML reader is quite versatile, and can be further
    expanded by adding custom Tag classes or extraction functions that interact directly with
    BeautifulSoup nodes.

    The Reader is suitable for datasets where each file should be extracted as a single
    document, or ones where each file contains multiple documents.

    In addition to generic extractor classes, this reader supports the `XML` extractor.

    Attributes:
        tag_toplevel: the top-level tag to search from in source documents.
        tag_entry: the tag that corresponds to a single document entry in source
            documents.
        external_file_tag_toplevel: the top-level tag to search from in external
            documents (if that functionality is used)

    '''

    tag_toplevel: TagSpecification = CurrentTag()
    '''
    The top-level tag in the source documents.

    Can be:

    - An XMLTag
    - A callable that takes the metadata of the document as input and returns an
        XMLTag.
    '''

    tag_entry: TagSpecification = CurrentTag()
    '''
    The tag that corresponds to a single document entry.

    Can be:

    - An XMLTag
    - A callable that takes the metadata of the document as input and returns an
        XMLTag
    '''

    external_file_tag_toplevel: TagSpecification = CurrentTag()
    '''
    The toplevel tag in external files (if you are using that functionality).

    Can be:

    - An XMLTag
    - A callable that takes the metadata of the document as input and returns an
        XMLTag. The metadata dictionary includes the values of "regular" fields for
        the document.
    '''

    def validate(self):
        # Make sure that extractors are sensible
        self._reject_extractors(extract.CSV)

    def iterate_data(self, data: bs4.BeautifulSoup, metadata: Dict) -> Iterable[Document]:
        external_soup = self._external_soup(metadata)

        # iterate through entries
        top_tag = resolve_tag_specification(self.__class__.tag_toplevel, metadata)
        bowl = top_tag.find_next_in_soup(data)

        if bowl:
            entry_tag = resolve_tag_specification(self.__class__.tag_entry, metadata)
            spoonfuls = entry_tag.find_in_soup(bowl)
            for spoon in spoonfuls:
                yield {
                    'soup_top': bowl,
                    'soup_entry': spoon,
                    'external_soup': external_soup,
                }
        else:
            logger.warning('Top-level tag not found')

    def extract_document(self, **document_data) -> Document:
        external_fields = self._external_fields()
        # fields should have unique names, but may not have stable instantiations
        # if FieldDefinitions are created on the fly, for example with class methods or @propertys
        external_fields_names = set(field.name for field in external_fields)
        regular_fields = [
            field for field in self.fields
            if field.name not in external_fields_names
        ]

        field_dict = {
            field.name: field.extractor.apply(**document_data)
            for field in regular_fields if not field.skip
        }

        external_soup = document_data.get('external_soup', None)
        metadata = document_data.get('metadata')

        if external_fields and external_soup:
            external_dict = self._external_source2dict(
                external_soup, external_fields, metadata | field_dict)
        else:
            external_dict = {
                field.name: None for field in external_fields if not field.skip
            }

        # yield the union of external fields and document fields
        return field_dict | external_dict

    def _external_fields(self) -> List[Field]:
        '''
        Subset of the reader's fields that rely on an external XML file.
        '''
        return [field for field in self.fields if
            isinstance(field.extractor, extract.XML) and field.extractor.external_file
        ]

    def _external_soup(self, metadata: Dict) -> Optional[bs4.BeautifulSoup]:
        '''
        Returns parsed tree for the external XML file, if applicable
        '''
        if any(self._external_fields()):
            if metadata and 'external_file' in metadata:
                return self.data_from_file(metadata['external_file'])
            else:
                logger.warning(
                    'Some fields have external_file property, but no external file is '
                    'provided in the source metadata'
                )

    def _external_source2dict(self, soup, external_fields: List[Field], metadata: Dict):
        '''
        given an external xml file with metadata,
        return a dictionary with tags which were found in that metadata
        wrt to the current source.
        '''
        tag = resolve_tag_specification(self.__class__.external_file_tag_toplevel, metadata)
        bowl = tag.find_next_in_soup(soup)

        if not bowl:
            logger.warning(
                'Top-level tag not found in `{}`'.format(metadata['external_file']))
            return {field.name: None for field in external_fields if not field.skip}

        return {
            field.name: field.extractor.apply(
                soup_top=bowl, soup_entry=bowl, metadata=metadata
            )
            for field in external_fields if not field.skip
        }

    def data_from_file(self, filename: str) -> bs4.BeautifulSoup:
        '''
        Returns beatifulsoup soup object for a given xml file
        '''
        # Loading XML
        logger.info('Reading XML file {} ...'.format(filename))
        with open(filename, 'rb') as f:
            data = f.read()
        logger.info('Loaded {} into memory...'.format(filename))
        return self.data_from_bytes(data)

    def data_from_bytes(self, data: bytes) -> bs4.BeautifulSoup:
        '''
        Parses content of a xml file
        '''
        return bs4.BeautifulSoup(data, 'lxml-xml')

    def data_from_response(self, data: Response) -> bs4.BeautifulSoup:
        return bs4.BeautifulSoup(data.content, 'lxml-xml')

class HTMLReader(XMLReader):
    '''
    An HTML reader extracts data from HTML sources.

    It is based on the XMLReader and supports the same options (`tag_toplevel` and
    `tag_entry`).

    In addition to generic extractor classes, this reader supports the `XML` extractor.
    '''

    def validate(self):
        self._reject_extractors(extract.CSV)


    def data_from_file(self, filename: str) -> bs4.BeautifulSoup:
        logger.info('Reading HTML file {} ...'.format(filename))
        with open(filename, 'rb') as f:
            data = f.read()
        # Parsing HTML
        soup = bs4.BeautifulSoup(data, 'html.parser')
        logger.info('Loaded {} into memory ...'.format(filename))
        return soup


    def iterate_data(self, data: bs4.BeautifulSoup, metadata: Dict) -> Iterable[Document]:
        # Extract fields from soup
        tag0 = self.tag_toplevel
        tag = self.tag_entry

        bowl = tag0.find_next_in_soup(data) if tag0 else data

        # if there is a entry level tag; with html this is not always the case
        if bowl and tag:
            for spoon in tag.find_in_soup(data):
                # yield
                yield {'soup_top': bowl, 'soup_entry': spoon}
        else:
            # yield all page content
            yield {'soup_entry': data}

class RDFReader(Reader):
    '''
    A base class for Readers of Resource Description Framework files.
    These could be in Turtle, JSON-LD, RDFXML or other formats,
    see [rdflib parsers](https://rdflib.readthedocs.io/en/stable/plugin_parsers.html).
    '''

    def validate(self):
        self._reject_extractors(extract.CSV, extract.XML, extract.JSON)


    # TODO: we could also allow Response as source data here, but that would mean the response would also need to include information of the data format, see [this example](https://github.com/RDFLib/rdflib/blob/4.1.2/rdflib/graph.py#L209)

    def data_from_file(self, path) -> Graph:
        ''' Read a RDF file as indicated by source, return a graph 
        Override this function to parse multiple source files into one graph

        Parameters:
            path: the name of the file to be parsed

        Returns:
            rdflib Graph object
        '''
        logger.info(f"parsing {path}")
        g = Graph()
        g.parse(path)
        return g


    def iterate_data(self, data: Graph, metadata: Dict) -> Iterable[Document]:
        document_subjects = self.document_subjects(data)
        for subject in document_subjects:
            yield {'graph': data, 'subject': subject}


    def document_subjects(self, graph: Graph) -> Iterable[Union[BNode, Literal, URIRef]]:
        ''' Override this function to return all subjects (i.e., first part of RDF triple) 
        with which to search for data in the RDF graph.
        Typically, such subjects are identifiers or urls.

        Parameters:
            graph: the graph to parse

        Returns:
            generator or list of nodes
        '''
        return graph.subjects()

class JSONReader(Reader):
    '''
    A base class for Readers of JSON encoded data.

    The reader can either be used on a collection of JSON files (`single_document=True`), in which each file represents a document,
    or for a JSON file containing lists of documents.

    If the attributes `record_path` and `meta` are set, they are used as arguments to [pandas.json_normalize](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.json_normalize.html) to unnest the JSON data.

    Attributes:
        single_document: indicates whether the data is organized such that a file represents a single document
        record_path: a path or list of paths by which a list of documents can be extracted from a large JSON file; irrelevant if `single_document = True`
        meta: a list of paths, or list of lists of paths, by which metadata common for all documents can be located; irrelevant if `single_document = True`
    """

    Examples:
        ### Multiple documents in one file:
        ```python
        example_data = {
            'path': {
                'sketch': 'Hungarian Phrasebook',
                'episode': 25,
                'to': {
                    'records':
                        [
                            {'speech': 'I will not buy this record. It is scratched.', 'character': 'tourist'},
                            {'speech': "No sir. This is a tobacconist's.", 'character': 'tobacconist'}
                        ]
                }
            }
        }

        MyJSONReader(JSONReader):
            record_path = ['path', 'to', 'records']
            meta = [['path', 'sketch'], ['path', 'episode']]

            speech = Field('speech', JSON('speech'))
            character = Field('character', JSON('character'))
            sketch = Field('sketch', JSON('path.sketch'))
            episode = Field('episode', JSON('path.episode'))
        ```
        To define the paths used to extract the field values, consider the dataformat the `pandas.json_normalize` creates:
        a table with each row representing a document, and columns corresponding to paths, either relative to documents within `record_path`,
        or relative to the top level (`meta`), with list of paths indicated by dots.
        ```csv
        row,speech,character,path.sketch,path.episode
        0,"I will not buy this record. It is scratched.","tourist","Hungarian Phrasebook",25
        1,"No sir. This is a tobacconist's.","tobacconist","Hungarian Phrasebook",25
        ```

        ### Single document per file:
        ```python
        example_data = {
            'sketch': 'Hungarian Phrasebook',
            'episode': 25,
            'scene': {
                'character': 'tourist',
                'speech': 'I will not buy this record. It is scratched.'
            }
        }

        MyJSONReader(JSONReader):
            single_document = True

            speech = Field('speech', JSON('scene', 'speech'))
            character = Field('character', JSON('scene', 'character))
            sketch = Field('sketch', JSON('sketch'))
            episode = Field('episode', JSON('episode))
        ```

    '''

    single_document: bool = False
    '''
    set to `True` if the data is structured such that one document is encoded in one .json file
    in that case, the reader assumes that there are no lists in such a file
    '''

    record_path: Optional[List[str]] = None
    '''
    a keyword or list of keywords by which a list of documents can be extracted from a large JSON file.
    Only relevant if `single_document=False`.
    '''

    meta: Optional[List[Union[str, List[str]]]] = None
    '''
    a list of keywords, or list of lists of keywords, by which metadata for each document can be located,
    if it is in a different path than `record_path`. Only relevant if `single_document=False`.
    '''

    def validate(self):
        self._reject_extractors(extract.XML, extract.CSV, extract.RDF)


    def iterate_data(self, data, metadata):
        if not self.single_document:
            documents = json_normalize(
                data, record_path=self.record_path, meta=self.meta
            ).to_dict('records')
        else:
            documents = [data]

        for doc in documents:
            yield {'data': doc}


    def data_from_file(self, path):
        with open(path, "r") as f:
            data = json.load(f)
        return data


    def data_from_bytes(self, bytes):
        return json.loads(bytes)


    def data_from_response(self, response):
        return response.json()

class Backup(Extractor):
    '''
    Try all given extractors in order and return the first result that evaluates as true

    This is a generic extractor that can be used in any `Reader`.

    Example usage:

        Backup(Constant(None), Constant('foo'))

    Since the first extractor returns `None`, the second extractor will be used, and the 
    extracted value would be `'foo'`.

    Note the difference with `Choice`: `Backup` is based on the _extracted value_,
    `Choice` on the `applicable` parameter of each extractor.

    Parameters:
        *extractors: extractors to use. These should be listed in descending order of
            preference.
        **kwargs: additional options to pass on to `Extractor`.
    '''
    def __init__(self, *extractors: Extractor, **kwargs):
        self.extractors = list(extractors)
        super().__init__(**kwargs)

    def _apply(self, *nargs, **kwargs):
        for extractor in self.extractors:
            result = extractor.apply(*nargs, **kwargs)
            if result:
                return result
        return None

class CSV(Extractor):
    '''
    This extractor extracts values from a list of CSV or spreadsheet rows.

    It should be used in readers based on `CSVReader` or `XLSXReader`.

    Parameters:
        column: The name of the column from which to extract the value.
        multiple: If a document spans multiple rows, the extracted value for a
            field with `multiple = True` is a list of the value in each row. If
            `multiple = False` (default), only the value from the first row is extracted.
        convert_to_none: optional, default is `['']`. Listed values are converted to
            `None`. If `None`/`False`, nothing is converted.
        **kwargs: additional options to pass on to `Extractor`.
    '''
    def __init__(self,
            column: str,
            multiple: bool = False,
            convert_to_none: List[str] = [''],
            *nargs, **kwargs):
        self.field = column
        self.multiple = multiple
        self.convert_to_none = convert_to_none or []
        super().__init__(*nargs, **kwargs)

    def _apply(self, rows, *nargs, **kwargs):
        if self.field in rows[0]:
            if self.multiple:
                return [self.format(row[self.field]) for row in rows]
            else:
                row = rows[0]
                return self.format(row[self.field])

    def format(self, value):
        if value and value not in self.convert_to_none:
            return value

class Cache(Extractor):
    '''
    Can be wrapped around another extractor to prevent repeatedly extracting the same
    value. 

    Makes an assumption the value of the extractor is going to be the same within a
    document, a source file, or even across the whole dataset.

    Parameters:
        extractor: Extractor of which the value is returned and cached.
        level: The level at which values should be cached. Can be `'document'`,
            `'source'`, or `'reader'`.
        **kwargs: additional options to pass on to `Extractor`

    Note: caching is based on the extractor instance and will not work across instances.
    For instance, in the example below, there would be no caching across fields.

    ```python
    fields = [
        Field(name='foo', extractor=Cache(XML('baz'))),
        Field(name='bar', extractor=Cache(XML('baz')))
    ]
    ```

    You could rewrite this as follows, so the XML tree is only queried once per document:

    ```python
    _my_extractor = Cache(XML('baz'))

    fields = [
        Field(name='foo', extractor=_my_extractor),
        Field(name='bar', extractor=_my_extractor)
    ]
    ```

    There is a similar issue when you use `@property` to define the `fields` of the
    reader.
    '''

    def __init__(self, extractor: Extractor, level: str = 'document', **kwargs):
        self.extractor = extractor
        self.level = level
        self.kwargs = {}
        super().__init__(**kwargs)

    def _apply(self, **kwargs):
        self.kwargs = kwargs

        if self.level == 'document':
            cache_params = [kwargs['source_index'], kwargs['index']]
        if self.level == 'source':
            cache_params = [kwargs['source_index']]
        if self.level == 'reader':
            cache_params = []

        return self._apply_cached(*cache_params)

    @lru_cache(maxsize=1)
    def _apply_cached(self, *cache_parameters):
        return self.extractor.apply(**self.kwargs)

class Choice(Extractor):
    '''
    Use the first applicable extractor from a list of extractors.

    This is a generic extractor that can be used in any `Reader`.

    The Choice extractor will use the `applicable` property of its provided extractors
    to check which applies. 

    Example usage: 

        Choice(Constant('foo', applicable=some_condition), Constant('bar'))

    This would extract `'foo'` if `some_condition` is met; otherwise,
    the extracted value will be `'bar'`.

    Note the difference with `Backup`: `Backup` will select the first truthy value from a
    list of extractors, but `Choice` only checks the `applicable` condition. For example:

        Choice(
            CSV('foo', applicable=Metadata('bar')),
            CSV('baz'),
        )

        Backup(
            CSV('foo', applicable=Metadata('bar')),
            CSV('baz'),
        )

    These extractors behave differently if the "bar" condition holds, but the "foo" field
    is empty. `Backup` will try to extract the "baz" field, `Choice` will not.

    Parameters:
        *extractors: extractors to choose from. These should be listed in descending
            order of preference.
        **kwargs: additional options to pass on to `Extractor`.
    '''

    def __init__(self, *extractors: Extractor, **kwargs):
        self.extractors = list(extractors)
        super().__init__(**kwargs)

    def _apply(self, *nargs, **kwargs):
        for extractor in self.extractors:
            if extractor._is_applicable(*nargs, **kwargs):
                return extractor.apply(*nargs, **kwargs)
        return None

class Combined(Extractor):
    '''
    Apply all given extractors and return the results as a tuple.

    This is a generic extractor that can be used in any `Reader`.

    Example usage:

        Combined(Constant('foo'), Constant('bar'))

    This would extract `('foo', 'bar')` for each document.

    Parameters:
        *extractors: extractors to combine.
        **kwargs: additional options to pass on to `Extractor`.
    '''

    def __init__(self, *extractors: Extractor, **kwargs):
        self.extractors = list(extractors)
        super().__init__(**kwargs)

    def _apply(self, *nargs, **kwargs):
        return tuple(
            extractor.apply(*nargs, **kwargs) for extractor in self.extractors
        )

class Constant(Extractor):
    '''
    This extractor 'extracts' the same value every time, regardless of input.

    This is a generic extractor that can be used in any `Reader`.

    It is especially useful in combination with `Backup` or `Choice`.

    Parameters:
        value: the value that should be "extracted".
        **kwargs: additional options to pass on to `Extractor`.
    '''

    def __init__(self, value: Any, *nargs, **kwargs):
        self.value = value
        super().__init__(*nargs, **kwargs)

    def _apply(self, *nargs, **kwargs):
        return self.value

class ExternalFile(Extractor):
    '''
    Free for all external file extractor that provides a stream to `stream_handler`
    to do whatever is needed to extract data from an external file. Relies on `associated_file`
    being present in the metadata. Note that the XMLExtractor has a built in trick to extract
    data from external files (i.e. setting `external_file`), so you probably need that if your
    external file is XML.

    Parameters:
        stream_handler: function that will handle the opened file.
        **kwargs: additional options to pass on to `Extractor`.
    '''

    def __init__(self, stream_handler: Callable, *nargs, **kwargs):
        super().__init__(*nargs, **kwargs)
        self.stream_handler = stream_handler

    def _apply(self, metadata, *nargs, **kwargs):
        '''
        Extract `associated_file` from metadata and call `self.stream_handler` with file stream.
        '''
        return self.stream_handler(open(metadata['associated_file'], 'r'))