diff --git a/README.md b/README.md index f3c34cc..b2f2f0c 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ ### The advanced feature: * Icons in my favorites list 'stations.yml' (the icon URL can be appended after the pipe character '|') * recently visited radio stations are stored in /.yast/resently.yml (compatible with stations.yml, for easy editing of your favorites and pasting into stations.yml) -* global filter configurable file ./ycast/filter.yml (with this you can globally reduce the radio stations according to your interests) +* global filter/limits configurable file ./ycast/filter.yml (with this you can globally reduce the radio stations according to your interests). The filter can be modified at runtime useing a REST API (/control/filter...), see below. * 5 frequently used radio stations can be selected on the target page (self-learning algorithm based on frequency of station selection) YCast is a self hosted replacement for the vTuner internet radio service which many AVRs use. @@ -145,6 +145,20 @@ Category two name: You can also have a look at the provided [example](examples/stations.yml.example) to better understand the configuration. +### Filter/limits +The filter configuration file .ycast/filter.yml (see example) allows to filter stations based on a whitelist / blacklist. The contents of this list specifies which attributes to filter on. + +The limits allow to filter out genres, countries and languages that fail to have a certain amount of items. It also sets the default station limit and allows to show or hide broken stations. Defaults are as follows: +* MINIMUM_COUNT_GENRE : 40 +* MINIMUM_COUNT_COUNTRY : 5 +* MINIMUM_COUNT_LANGUAGE : 5 +* DEFAULT_STATION_LIMIT : 200 +* SHOW_BROKEN_STATIONS : False + +You can set your own valies in .ycast/filter.xml by adding these attributes and there values in the limits list. The filter file is not reread automatically when modified while the server is running. Send a HUP signal to trigger but it's preferred to use the api (see below) to modify the lists. + +The current filters/limits can be queried through a REST API by calling the GET method on /control/filter/whitelist, /control/filter/blacklist and /control/filter/limits. They can be modified by using the POST method an posting a JSON with the items to modify. Specifying a null value for an item will delete it from the list or, in the case of the limits, reset it to its default. + ## Firewall rules * Your AVR needs access to the internet. diff --git a/examples/filter.yml.example b/examples/filter.yml.example index ce4fd1f..b0eb950 100644 --- a/examples/filter.yml.example +++ b/examples/filter.yml.example @@ -1,6 +1,7 @@ blacklist: favicon: '' whitelist: + country: Germany countrycode: DE,US,NO,GB languagecodes: en,no lastcheckok: 1 diff --git a/ycast/__main__.py b/ycast/__main__.py index 8826c79..c75da35 100755 --- a/ycast/__main__.py +++ b/ycast/__main__.py @@ -3,12 +3,18 @@ import argparse import logging import sys +import signal from ycast import __version__ from ycast import server logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.INFO) +def handler(signum, frame): + logging.info('Signal received: rereading filter config') + from ycast.my_filter import init_filter_file + init_filter_file() +signal.signal(signal.SIGHUP, handler) def launch_server(): parser = argparse.ArgumentParser(description='vTuner API emulation') @@ -27,8 +33,8 @@ def launch_server(): # initialize important ycast parameters from ycast.generic import init_base_dir init_base_dir('/.ycast') - from ycast.my_filter import init_limits_and_filters - init_limits_and_filters() + from ycast.my_filter import init_filter_file + init_filter_file() server.run(arguments.config, arguments.address, arguments.port) diff --git a/ycast/my_filter.py b/ycast/my_filter.py index d612830..baa40f2 100644 --- a/ycast/my_filter.py +++ b/ycast/my_filter.py @@ -3,53 +3,39 @@ import logging from ycast import generic from ycast.generic import get_json_attr -white_list = {} +white_list = {'lastcheckok': 1} black_list = {} -filter_dictionary = {} +limit_list = {} +limit_defs_int ={ 'MINIMUM_COUNT_GENRE' : 40, 'MINIMUM_COUNT_COUNTRY' : 5, 'MINIMUM_COUNT_LANGUAGE' : 5, 'DEFAULT_STATION_LIMIT' : 200} +limit_defs_bool ={ 'SHOW_BROKEN_STATIONS' : False} parameter_failed_list = {} count_used = 0 count_hit = 0 -LIMITS_NAME = 'limits' -MINIMUM_COUNT_GENRE = 40 -MINIMUM_COUNT_COUNTRY = 5 -MINIMUM_COUNT_LANGUAGE = 5 -DEFAULT_STATION_LIMIT = 200 -SHOW_BROKEN_STATIONS = False - - -def init_limits_and_filters(): - global MINIMUM_COUNT_GENRE, MINIMUM_COUNT_LANGUAGE, MINIMUM_COUNT_COUNTRY, DEFAULT_STATION_LIMIT, SHOW_BROKEN_STATIONS - logging.info('Initialize Limits and Filters') - init_filter_file() - MINIMUM_COUNT_GENRE = get_limit('MINIMUM_COUNT_GENRE', 40) - MINIMUM_COUNT_COUNTRY = get_limit('MINIMUM_COUNT_COUNTRY', 5) - MINIMUM_COUNT_LANGUAGE = get_limit('MINIMUM_COUNT_LANGUAGE', 5) - DEFAULT_STATION_LIMIT = get_limit('DEFAULT_STATION_LIMIT', 200) - SHOW_BROKEN_STATIONS = get_limit('SHOW_BROKEN_STATIONS', False) - def init_filter_file(): - global filter_dictionary, white_list, black_list + global white_list, black_list, limit_list + logging.info('Reading Limits and Filters') filter_dictionary = generic.read_yaml_file(generic.get_filter_file()) - is_updated = False if filter_dictionary is None: filter_dictionary = {} is_updated = True if 'whitelist' in filter_dictionary: - white_list = filter_dictionary['whitelist'] - else: - white_list = {'lastcheckok': 1} - is_updated = True + # Copy dict items to keep the 1 default item + for f in filter_dictionary['whitelist']: + white_list[f]=filter_dictionary['whitelist'][f] if 'blacklist' in filter_dictionary: + # reference, no defaults black_list = filter_dictionary['blacklist'] - else: - black_list = {} - is_updated = True - if is_updated: - filter_dictionary = {'whitelist': white_list, 'blacklist': black_list} - generic.write_yaml_file(generic.get_var_path() + '/filter.yml', filter_dictionary) + if 'limits' in filter_dictionary: + set_limits(filter_dictionary['limits']) + +def write_filter_config(): + global limit_list + filter_dictionary = {'whitelist': white_list, 'blacklist': black_list} + if len(limit_list) > 0: filter_dictionary['limits']=limit_list + generic.write_yaml_file(generic.get_var_path() + '/filter.yml', filter_dictionary) def begin_filter(): global parameter_failed_list @@ -58,7 +44,7 @@ def begin_filter(): count_used = 0 count_hit = 0 - init_filter_file() + #init_filter_file() parameter_failed_list.clear() return @@ -138,17 +124,31 @@ def check_station(station_json): return True -def get_limit(param_name, default): - global filter_dictionary - tempdict = generic.read_yaml_file(generic.get_var_path() + '/filter.yml') - if tempdict is not None: - filter_dictionary = tempdict - limits_dict = {} - if LIMITS_NAME in filter_dictionary: - limits_dict = filter_dictionary[LIMITS_NAME] - if param_name in limits_dict: - return limits_dict[param_name] - limits_dict[param_name] = default - filter_dictionary[LIMITS_NAME] = limits_dict - generic.write_yaml_file(generic.get_var_path() + '/filter.yml', filter_dictionary) - return default +def get_limit(param_name): + global limit_defs + if param_name in limit_defs_int: return limit_list.get(param_name,limit_defs_int[param_name]) + if param_name in limit_defs_bool: return limit_list.get(param_name,limit_defs_bool[param_name]) + else: return None + +def get_limit_list(): + global limit_defs_int, limit_defs_bool + my_limits={} + limit_defs=dict(limit_defs_int) + limit_defs.update(limit_defs_bool) + for l in limit_defs: + my_limits[l]=get_limit(l) + return my_limits + +def set_limits(limits): + global limit_list, limit_defs_int, limit_defs_bool + for l in limits: + if limits[l] == None: + limit_list.pop(l, None) + elif l in limit_defs_int: + if isinstance(limits[l], int) and limits[l] > 0: + limit_list[l]=limits[l] + elif l in limit_defs_bool: + if isinstance(limits[l], bool): limit_list[l]=limits[l] + else: + loggin.error("Invalid limit %s") % l + return get_limit_list() diff --git a/ycast/radiobrowser.py b/ycast/radiobrowser.py index eff4370..26a3dfe 100644 --- a/ycast/radiobrowser.py +++ b/ycast/radiobrowser.py @@ -8,8 +8,7 @@ import logging from ycast import __version__, my_filter import ycast.vtuner as vtuner import ycast.generic as generic -from ycast.my_filter import check_station, begin_filter, end_filter, SHOW_BROKEN_STATIONS, MINIMUM_COUNT_COUNTRY, \ - MINIMUM_COUNT_LANGUAGE, MINIMUM_COUNT_GENRE, DEFAULT_STATION_LIMIT +from ycast.my_filter import check_station, begin_filter, end_filter, get_limit from ycast.generic import get_json_attr API_ENDPOINT = "http://all.api.radio-browser.info" @@ -92,12 +91,12 @@ def get_country_directories(): begin_filter() country_directories = [] apicall = 'countries' - if not SHOW_BROKEN_STATIONS: + if not get_limit('SHOW_BROKEN_STATIONS'): apicall += '?hidebroken=true' countries_raw = request(apicall) for country_raw in countries_raw: if get_json_attr(country_raw, 'name') and get_json_attr(country_raw, 'stationcount') and \ - int(get_json_attr(country_raw, 'stationcount')) > MINIMUM_COUNT_COUNTRY: + int(get_json_attr(country_raw, 'stationcount')) > get_limit('MINIMUM_COUNT_COUNTRY'): if my_filter.chk_parameter('country', get_json_attr(country_raw, 'name')): country_directories.append(generic.Directory(get_json_attr(country_raw, 'name'), get_json_attr(country_raw, 'stationcount'))) @@ -108,12 +107,12 @@ def get_language_directories(): begin_filter() language_directories = [] apicall = 'languages' - if not SHOW_BROKEN_STATIONS: + if not get_limit('SHOW_BROKEN_STATIONS'): apicall += '?hidebroken=true' languages_raw = request(apicall) for language_raw in languages_raw: if get_json_attr(language_raw, 'name') and get_json_attr(language_raw, 'stationcount') and \ - int(get_json_attr(language_raw, 'stationcount')) > MINIMUM_COUNT_LANGUAGE: + int(get_json_attr(language_raw, 'stationcount')) > get_limit('MINIMUM_COUNT_LANGUAGE'): if my_filter.chk_parameter('languagecodes', get_json_attr(language_raw, 'iso_639')): language_directories.append(generic.Directory(get_json_attr(language_raw, 'name'), get_json_attr(language_raw, 'stationcount'), @@ -124,12 +123,12 @@ def get_language_directories(): def get_genre_directories(): genre_directories = [] apicall = 'tags' - if not SHOW_BROKEN_STATIONS: + if not get_limit('SHOW_BROKEN_STATIONS'): apicall += '?hidebroken=true' genres_raw = request(apicall) for genre_raw in genres_raw: if get_json_attr(genre_raw, 'name') and get_json_attr(genre_raw, 'stationcount') and \ - int(get_json_attr(genre_raw, 'stationcount')) > MINIMUM_COUNT_GENRE: + int(get_json_attr(genre_raw, 'stationcount')) > get_limit('MINIMUM_COUNT_GENRE'): genre_directories.append(generic.Directory(get_json_attr(genre_raw, 'name'), get_json_attr(genre_raw, 'stationcount'), get_json_attr(genre_raw, 'name').capitalize())) @@ -179,7 +178,7 @@ def get_stations_by_genre(genre): return stations -def get_stations_by_votes(limit=DEFAULT_STATION_LIMIT): +def get_stations_by_votes(limit=get_limit('DEFAULT_STATION_LIMIT')): begin_filter() station_cache.clear() stations = [] @@ -193,7 +192,7 @@ def get_stations_by_votes(limit=DEFAULT_STATION_LIMIT): return stations -def search(name, limit=DEFAULT_STATION_LIMIT): +def search(name, limit=get_limit('DEFAULT_STATION_LIMIT')): begin_filter() station_cache.clear() stations = [] diff --git a/ycast/server.py b/ycast/server.py index 7211a16..a22ece4 100644 --- a/ycast/server.py +++ b/ycast/server.py @@ -9,6 +9,7 @@ import ycast.radiobrowser as radiobrowser import ycast.my_stations as my_stations import ycast.generic as generic import ycast.station_icons as station_icons +import ycast.my_filter as my_filter from ycast import my_recentlystation from ycast.my_recentlystation import signal_station_selected @@ -135,6 +136,37 @@ def upstream(path): logging.error("Unhandled upstream query (/setupapp/%s)", path) abort(404) +@app.route('/control/filter/', + methods=['POST','GET']) +def set_filters(item): + update_limits=False + # POST updates the whitelist or blacklist, GET just returns the current attributes/values. + myfilter={} + if item.endswith('blacklist'): + myfilter=my_filter.black_list + if item.endswith('whitelist'): + myfilter=my_filter.white_list + if item.endswith('limits'): + myfilter=my_filter.get_limit_list() + update_limits=True + if request.method == 'POST': + content_type = request.headers.get('Content-Type') + if (content_type == 'application/json'): + json = request.json + else: + return abort(400,'Content-Type not supported!: ' + item) + if update_limits: + myfilter=my_filter.set_limits(json) + else: + for j in json: + # Attribute with null value removes item from the list otherwise add the attribute or update the value + if json[j] == None: + myfilter.pop(j, None) + else: + myfilter[j]=json[j] + my_filter.write_filter_config() + json=flask.jsonify(myfilter) + return json @app.route('/api/', methods=['GET', 'POST']) diff --git a/ycast/test_YCast.py b/ycast/test_YCast.py index 88e1375..af7d013 100644 --- a/ycast/test_YCast.py +++ b/ycast/test_YCast.py @@ -12,7 +12,7 @@ from ycast import my_filter, generic, radiobrowser, my_recentlystation class MyTestCase(unittest.TestCase): logging.getLogger().setLevel(logging.DEBUG) - generic.init_base_dir("../../.test_ycast") + generic.init_base_dir("/.test_ycast") my_filter.init_filter_file() @@ -34,10 +34,10 @@ class MyTestCase(unittest.TestCase): def test_init_filter(self): my_filter.begin_filter() - - for elem in my_filter.filter_dictionary: + filter_dictionary={ "whitelist" : my_filter.white_list, "blacklist" : my_filter.black_list} + for elem in filter_dictionary: logging.warning("Name filtertype: %s", elem) - filter_param = my_filter.filter_dictionary[elem] + filter_param = filter_dictionary[elem] if filter_param: for par in filter_param: logging.warning(" Name paramter: %s", par) @@ -51,21 +51,29 @@ class MyTestCase(unittest.TestCase): def test_get_languages(self): result = radiobrowser.get_language_directories() - assert len(result) == 3 + logging.info("Languages (%d)", len(result)) + # Based on the example filter is should yield 2 + assert len(result) == 2 def test_get_countries(self): my_filter.init_filter_file() result = radiobrowser.get_country_directories() - assert len(result) == 137 + logging.info("Countries (%d)", len(result)) + # Test for Germany only 1, nach der Wiedervereinigung... + assert len(result) == 1 def test_get_genre(self): result = radiobrowser.get_genre_directories() - assert len(result) < 300 + logging.info("Genres (%d)", len(result)) + # Not a useful test, changes all the time + #assert len(result) < 300 def test_get_limits(self): - result = my_filter.get_limit('irgendwas',20) - assert result == 20 + result = my_filter.get_limit('MINIMUM_COUNT_COUNTRY') + assert result == 5 + result = my_filter.get_limit('SHOW_BROKEN_STATIONS') + assert result == False def test_recently_hit(self):