import re
from pomp.core.base import BaseCrawler


python_sentence_re = re.compile('[\w\s]{0,}python[\s\w]{0,}', re.I | re.M)


class MyCrawler(BaseCrawler):
    """Extract all sentences with `python` word"""
    ENTRY_REQUESTS = 'http://python.org/news'  # อ้างอิงหน้าเว็บที่ต้องการ crawl

    def extract_items(self, response):
        for i in python_sentence_re.findall(response.body.decode('utf-8')): # ถอดหน้าเว็บเป็น UTF 8
            item = i.strip()
            print(item)
            return item

    def next_requests(self, response):
        return None  # crawler เพจเดียว, แล้วหยุด crawl ต่อ


if __name__ == '__main__':
    from pomp.core.engine import Pomp
    from pomp.contrib.urllibtools import UrllibDownloader

    pomp = Pomp(
        downloader=UrllibDownloader(),
    )

    pomp.pump(MyCrawler()) 

"""
Extract python news from python.org
"""
import sys
import re
import logging
from pomp.core.base import BaseCrawler, BasePipeline
from pomp.core.item import Item, Field
from pomp.contrib.urllibtools import UrllibDownloader


logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
news_re = re.compile(
    r'

(.*?)
([\s\S]*?)

(.*?)
')


class PythonNewsItem(Item):
    title = Field()
    published = Field()

    def __repr__(self):
        return '%s\n\t%s\n' % (
            self.title,
            self.published,
        )


class PythonNewsCrawler(BaseCrawler):
    ENTRY_REQUESTS = 'https://python.org/news/'

    def extract_items(self, response):
        for i in news_re.findall(response.body.decode('utf-8')):
            item = PythonNewsItem()
            item.title, item.published = i[0], i[2]
            yield item

    def next_requests(self, response):
        return None  # one page crawler


class PrintPipeline(BasePipeline):

    def process(self, crawler, item):
        print(item)
        return item


if __name__ == '__main__':
    from pomp.core.engine import Pomp

    pomp = Pomp(
        downloader=UrllibDownloader(),
        pipelines=[PrintPipeline()],
    )

    pomp.pump(PythonNewsCrawler())
input()