Module pywander.crawler.web_utils

Functions

def use_cache_callback_requests_web(cachedb, cache_data, func, args, kwargs, use_cache_oldest_dt=None)
Expand source code
def use_cache_callback_requests_web(cachedb, cache_data, func, args, kwargs,
                                    use_cache_oldest_dt=None):
    timestamp = cache_data.get('timestamp', timestamp_current())
    data_dt = timestamp_to_dt(timestamp)

    if use_cache_oldest_dt is None:
        target_dt = dt_current() - relativedelta(seconds=14)  # default 14 days
    else:
        target_dt = use_cache_oldest_dt

    if data_dt.tzinfo is None:
        data_dt.replace(tzinfo=timezone.utc)
    if target_dt.tzinfo is None:
        target_dt.replace(tzinfo=timezone.utc)

    if data_dt < target_dt:  # too old then we will re-excute the function
        t = threading.Thread(target=_update_requests_web,
                             args=(cachedb, cache_data, args))
        t.daemon = True
        t.start()