Filter/limits loaded at startup from config file. File is not reread automatically, use SIHUP to trigger. API to query/modify filters/limits.

README.md

@@ -13,7 +13,7 @@
 ### The advanced feature:
 * Icons in my favorites list 'stations.yml' (the icon URL can be appended after the pipe character '|')
 * recently visited radio stations are stored in /.yast/resently.yml (compatible with stations.yml, for easy editing of your favorites and pasting into stations.yml)
-* global filter configurable file ./ycast/filter.yml (with this you can globally reduce the radio stations according to your interests)
+* global filter/limits configurable file ./ycast/filter.yml (with this you can globally reduce the radio stations according to your interests). The filter can be modified at runtime useing a REST API (/control/filter...), see below.
 * 5 frequently used radio stations can be selected on the target page (self-learning algorithm based on frequency of station selection)
 
 YCast is a self hosted replacement for the vTuner internet radio service which many AVRs use.
@@ -145,6 +145,20 @@ Category two name:
 
 You can also have a look at the provided [example](examples/stations.yml.example) to better understand the configuration.
 
+### Filter/limits
+The filter configuration file .ycast/filter.yml (see example) allows to filter stations based on a whitelist / blacklist. The contents of this list specifies which attributes to filter on.
+
+The limits allow to filter out genres, countries and languages that fail to have a certain amount of items. It also sets the default station limit and allows to show or hide broken stations. Defaults are as follows:
+* MINIMUM_COUNT_GENRE : 40
+* MINIMUM_COUNT_COUNTRY : 5
+* MINIMUM_COUNT_LANGUAGE : 5
+* DEFAULT_STATION_LIMIT : 200
+* SHOW_BROKEN_STATIONS : False
+
+You can set your own valies in .ycast/filter.xml by adding these attributes and there values in the limits list. The filter file is not reread automatically when modified while the server is running. Send a HUP signal to trigger but it's preferred to use the api (see below) to modify the lists.
+
+The current filters/limits can be queried  through a REST API by calling the GET method on /control/filter/whitelist, /control/filter/blacklist and /control/filter/limits. They can be modified by using the POST method an posting a JSON with the items to modify. Specifying a null value for an item will delete it from the list or, in the case of the limits, reset it to its default.
+
 ## Firewall rules
 
  * Your AVR needs access to the internet.

examples/filter.yml.example

@@ -1,6 +1,7 @@
 blacklist:
   favicon: ''
 whitelist:
+  country: Germany
   countrycode: DE,US,NO,GB
   languagecodes: en,no
   lastcheckok: 1

ycast/__main__.py

@@ -3,12 +3,18 @@
 import argparse
 import logging
 import sys
+import signal
 
 from ycast import __version__
 from ycast import server
 
 logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.INFO)
 
+def handler(signum, frame):
+    logging.info('Signal received: rereading filter config')
+    from ycast.my_filter import init_filter_file
+    init_filter_file()
+signal.signal(signal.SIGHUP, handler)
 
 def launch_server():
     parser = argparse.ArgumentParser(description='vTuner API emulation')
@@ -27,8 +33,8 @@ def launch_server():
     # initialize important ycast parameters
     from ycast.generic import init_base_dir
     init_base_dir('/.ycast')
-    from ycast.my_filter import init_limits_and_filters
-    init_limits_and_filters()
+    from ycast.my_filter import init_filter_file
+    init_filter_file()
 
     server.run(arguments.config, arguments.address, arguments.port)
 

ycast/my_filter.py

@@ -3,53 +3,39 @@ import logging
 from ycast import generic
 from ycast.generic import get_json_attr
 
-white_list = {}
+white_list = {'lastcheckok': 1}
 black_list = {}
-filter_dictionary = {}
+limit_list = {}
+limit_defs_int ={ 'MINIMUM_COUNT_GENRE' : 40, 'MINIMUM_COUNT_COUNTRY' : 5, 'MINIMUM_COUNT_LANGUAGE' : 5, 'DEFAULT_STATION_LIMIT' : 200}
+limit_defs_bool ={ 'SHOW_BROKEN_STATIONS' : False}
 parameter_failed_list = {}
 count_used = 0
 count_hit = 0
 
-LIMITS_NAME = 'limits'
-MINIMUM_COUNT_GENRE = 40
-MINIMUM_COUNT_COUNTRY = 5
-MINIMUM_COUNT_LANGUAGE = 5
-DEFAULT_STATION_LIMIT = 200
-SHOW_BROKEN_STATIONS = False
-
-
-def init_limits_and_filters():
-    global MINIMUM_COUNT_GENRE, MINIMUM_COUNT_LANGUAGE, MINIMUM_COUNT_COUNTRY, DEFAULT_STATION_LIMIT, SHOW_BROKEN_STATIONS
-    logging.info('Initialize Limits and Filters')
-    init_filter_file()
-    MINIMUM_COUNT_GENRE = get_limit('MINIMUM_COUNT_GENRE', 40)
-    MINIMUM_COUNT_COUNTRY = get_limit('MINIMUM_COUNT_COUNTRY', 5)
-    MINIMUM_COUNT_LANGUAGE = get_limit('MINIMUM_COUNT_LANGUAGE', 5)
-    DEFAULT_STATION_LIMIT = get_limit('DEFAULT_STATION_LIMIT', 200)
-    SHOW_BROKEN_STATIONS = get_limit('SHOW_BROKEN_STATIONS', False)
-
 def init_filter_file():
-    global filter_dictionary, white_list, black_list
+    global white_list, black_list, limit_list
+    logging.info('Reading Limits and Filters')
     filter_dictionary = generic.read_yaml_file(generic.get_filter_file())
-    is_updated = False
     if filter_dictionary is None:
         filter_dictionary = {}
         is_updated = True
     if 'whitelist' in filter_dictionary:
-        white_list = filter_dictionary['whitelist']
-    else:
-        white_list = {'lastcheckok': 1}
-        is_updated = True
+        # Copy dict items to keep the 1 default item
+        for f in filter_dictionary['whitelist']:
+            white_list[f]=filter_dictionary['whitelist'][f]
 
     if 'blacklist' in filter_dictionary:
+        # reference, no defaults
         black_list = filter_dictionary['blacklist']
-    else:
-        black_list = {}
-        is_updated = True
 
-    if is_updated:
-        filter_dictionary = {'whitelist': white_list, 'blacklist': black_list}
-        generic.write_yaml_file(generic.get_var_path() + '/filter.yml', filter_dictionary)
+    if 'limits' in filter_dictionary:
+        set_limits(filter_dictionary['limits'])
+
+def write_filter_config():
+    global limit_list
+    filter_dictionary = {'whitelist': white_list, 'blacklist': black_list}
+    if len(limit_list) > 0: filter_dictionary['limits']=limit_list
+    generic.write_yaml_file(generic.get_var_path() + '/filter.yml', filter_dictionary)
 
 def begin_filter():
     global parameter_failed_list
@@ -58,7 +44,7 @@ def begin_filter():
     count_used = 0
     count_hit = 0
 
-    init_filter_file()
+    #init_filter_file()
 
     parameter_failed_list.clear()
     return
@@ -138,17 +124,31 @@ def check_station(station_json):
     return True
 
 
-def get_limit(param_name, default):
-    global filter_dictionary
-    tempdict = generic.read_yaml_file(generic.get_var_path() + '/filter.yml')
-    if tempdict is not None:
-        filter_dictionary = tempdict
-    limits_dict = {}
-    if LIMITS_NAME in filter_dictionary:
-        limits_dict = filter_dictionary[LIMITS_NAME]
-    if param_name in limits_dict:
-        return limits_dict[param_name]
-    limits_dict[param_name] = default
-    filter_dictionary[LIMITS_NAME] = limits_dict
-    generic.write_yaml_file(generic.get_var_path() + '/filter.yml', filter_dictionary)
-    return default
+def get_limit(param_name):
+    global limit_defs
+    if param_name in limit_defs_int: return limit_list.get(param_name,limit_defs_int[param_name])
+    if param_name in limit_defs_bool: return limit_list.get(param_name,limit_defs_bool[param_name])
+    else: return None
+
+def get_limit_list():
+    global limit_defs_int, limit_defs_bool
+    my_limits={}
+    limit_defs=dict(limit_defs_int)
+    limit_defs.update(limit_defs_bool)
+    for l in limit_defs:
+         my_limits[l]=get_limit(l)
+    return my_limits
+
+def set_limits(limits):
+    global limit_list, limit_defs_int, limit_defs_bool
+    for l in limits:
+        if limits[l] == None:
+            limit_list.pop(l, None)
+        elif l in limit_defs_int:
+            if isinstance(limits[l], int) and limits[l] > 0:
+                limit_list[l]=limits[l]
+        elif l in limit_defs_bool:
+            if isinstance(limits[l], bool): limit_list[l]=limits[l]
+        else:
+            loggin.error("Invalid limit %s") % l
+    return get_limit_list()

ycast/radiobrowser.py

@@ -8,8 +8,7 @@ import logging
 from ycast import __version__, my_filter
 import ycast.vtuner as vtuner
 import ycast.generic as generic
-from ycast.my_filter import check_station, begin_filter, end_filter, SHOW_BROKEN_STATIONS, MINIMUM_COUNT_COUNTRY, \
-    MINIMUM_COUNT_LANGUAGE, MINIMUM_COUNT_GENRE, DEFAULT_STATION_LIMIT
+from ycast.my_filter import check_station, begin_filter, end_filter, get_limit 
 from ycast.generic import get_json_attr
 
 API_ENDPOINT = "http://all.api.radio-browser.info"
@@ -92,12 +91,12 @@ def get_country_directories():
     begin_filter()
     country_directories = []
     apicall = 'countries'
-    if not SHOW_BROKEN_STATIONS:
+    if not get_limit('SHOW_BROKEN_STATIONS'):
         apicall += '?hidebroken=true'
     countries_raw = request(apicall)
     for country_raw in countries_raw:
         if get_json_attr(country_raw, 'name') and get_json_attr(country_raw, 'stationcount') and \
-                int(get_json_attr(country_raw, 'stationcount')) > MINIMUM_COUNT_COUNTRY:
+                int(get_json_attr(country_raw, 'stationcount')) > get_limit('MINIMUM_COUNT_COUNTRY'):
             if my_filter.chk_parameter('country', get_json_attr(country_raw, 'name')):
                 country_directories.append(generic.Directory(get_json_attr(country_raw, 'name'),
                                                              get_json_attr(country_raw, 'stationcount')))
@@ -108,12 +107,12 @@ def get_language_directories():
     begin_filter()
     language_directories = []
     apicall = 'languages'
-    if not SHOW_BROKEN_STATIONS:
+    if not get_limit('SHOW_BROKEN_STATIONS'):
         apicall += '?hidebroken=true'
     languages_raw = request(apicall)
     for language_raw in languages_raw:
         if get_json_attr(language_raw, 'name') and get_json_attr(language_raw, 'stationcount') and \
-                int(get_json_attr(language_raw, 'stationcount')) > MINIMUM_COUNT_LANGUAGE:
+                int(get_json_attr(language_raw, 'stationcount')) > get_limit('MINIMUM_COUNT_LANGUAGE'):
             if my_filter.chk_parameter('languagecodes', get_json_attr(language_raw, 'iso_639')):
                 language_directories.append(generic.Directory(get_json_attr(language_raw, 'name'),
                                                               get_json_attr(language_raw, 'stationcount'),
@@ -124,12 +123,12 @@ def get_language_directories():
 def get_genre_directories():
     genre_directories = []
     apicall = 'tags'
-    if not SHOW_BROKEN_STATIONS:
+    if not get_limit('SHOW_BROKEN_STATIONS'):
         apicall += '?hidebroken=true'
     genres_raw = request(apicall)
     for genre_raw in genres_raw:
         if get_json_attr(genre_raw, 'name') and get_json_attr(genre_raw, 'stationcount') and \
-                int(get_json_attr(genre_raw, 'stationcount')) > MINIMUM_COUNT_GENRE:
+                int(get_json_attr(genre_raw, 'stationcount')) > get_limit('MINIMUM_COUNT_GENRE'):
             genre_directories.append(generic.Directory(get_json_attr(genre_raw, 'name'),
                                                        get_json_attr(genre_raw, 'stationcount'),
                                                        get_json_attr(genre_raw, 'name').capitalize()))
@@ -179,7 +178,7 @@ def get_stations_by_genre(genre):
     return stations
 
 
-def get_stations_by_votes(limit=DEFAULT_STATION_LIMIT):
+def get_stations_by_votes(limit=get_limit('DEFAULT_STATION_LIMIT')):
     begin_filter()
     station_cache.clear()
     stations = []
@@ -193,7 +192,7 @@ def get_stations_by_votes(limit=DEFAULT_STATION_LIMIT):
     return stations
 
 
-def search(name, limit=DEFAULT_STATION_LIMIT):
+def search(name, limit=get_limit('DEFAULT_STATION_LIMIT')):
     begin_filter()
     station_cache.clear()
     stations = []

ycast/server.py

@@ -9,6 +9,7 @@ import ycast.radiobrowser as radiobrowser
 import ycast.my_stations as my_stations
 import ycast.generic as generic
 import ycast.station_icons as station_icons
+import ycast.my_filter as my_filter
 from ycast import my_recentlystation
 from ycast.my_recentlystation import signal_station_selected
 
@@ -135,6 +136,37 @@ def upstream(path):
     logging.error("Unhandled upstream query (/setupapp/%s)", path)
     abort(404)
 
+@app.route('/control/filter/<path:item>',
+           methods=['POST','GET'])
+def set_filters(item):
+    update_limits=False
+    # POST updates the whitelist or blacklist, GET just returns the current attributes/values.
+    myfilter={}
+    if item.endswith('blacklist'):
+        myfilter=my_filter.black_list
+    if item.endswith('whitelist'):
+        myfilter=my_filter.white_list
+    if item.endswith('limits'):
+        myfilter=my_filter.get_limit_list()
+        update_limits=True
+    if request.method == 'POST':
+        content_type = request.headers.get('Content-Type')
+        if (content_type == 'application/json'):
+            json = request.json
+        else:
+            return  abort(400,'Content-Type not supported!: ' + item)
+        if update_limits:
+            myfilter=my_filter.set_limits(json)
+        else:
+            for j in json:
+                # Attribute with null value removes item from the list otherwise add the attribute or update the value
+                if json[j] == None:
+                    myfilter.pop(j, None)
+                else:
+                    myfilter[j]=json[j]
+        my_filter.write_filter_config()
+    json=flask.jsonify(myfilter)
+    return json
 
 @app.route('/api/<path:path>',
            methods=['GET', 'POST'])

ycast/test_YCast.py

@@ -12,7 +12,7 @@ from ycast import my_filter, generic, radiobrowser, my_recentlystation
 class MyTestCase(unittest.TestCase):
 
     logging.getLogger().setLevel(logging.DEBUG)
-    generic.init_base_dir("../../.test_ycast")
+    generic.init_base_dir("/.test_ycast")
     my_filter.init_filter_file()
 
 
@@ -34,10 +34,10 @@ class MyTestCase(unittest.TestCase):
 
     def test_init_filter(self):
         my_filter.begin_filter()
-
-        for elem in my_filter.filter_dictionary:
+        filter_dictionary={ "whitelist" : my_filter.white_list, "blacklist" : my_filter.black_list}
+        for elem in filter_dictionary:
             logging.warning("Name filtertype: %s", elem)
-            filter_param = my_filter.filter_dictionary[elem]
+            filter_param = filter_dictionary[elem]
             if filter_param:
                 for par in filter_param:
                     logging.warning("    Name paramter: %s", par)
@@ -51,21 +51,29 @@ class MyTestCase(unittest.TestCase):
 
     def test_get_languages(self):
         result = radiobrowser.get_language_directories()
-        assert len(result) == 3
+        logging.info("Languages (%d)", len(result))
+        # Based on the example filter is should yield 2
+        assert len(result) == 2
 
     def test_get_countries(self):
         my_filter.init_filter_file()
 
         result = radiobrowser.get_country_directories()
-        assert len(result) == 137
+        logging.info("Countries (%d)", len(result))
+        # Test for Germany only 1, nach der Wiedervereinigung...
+        assert len(result) == 1
 
     def test_get_genre(self):
         result = radiobrowser.get_genre_directories()
-        assert len(result) < 300
+        logging.info("Genres (%d)", len(result))
+        # Not a useful test, changes all the time
+        #assert len(result) < 300
 
     def test_get_limits(self):
-        result = my_filter.get_limit('irgendwas',20)
-        assert result == 20
+        result = my_filter.get_limit('MINIMUM_COUNT_COUNTRY')
+        assert result == 5
+        result = my_filter.get_limit('SHOW_BROKEN_STATIONS')
+        assert result == False
 
     def test_recently_hit(self):