Update models, functions and Readme

README.md

@@ -1,23 +1,47 @@
 ## weather_api
 
-### Datacose Weather API
+### Overview
+This API serves as a backend system for managing the location of the Open Meteo Weather dashboard. It includes functionalities for creating, updating, deleting and searching available locations.
+
+
+### Endpoints
+1. **GET /locations**: 
+	* Retrieve a list of all the locations stored in the database.
+	* The locations configured by the user
+2. **GET /locations/{id}**:
+	* Retriev weather data for the specified ID.
+3. **POST /locations**: Add a new location to the database.
+4. **DELETE /locations/{id}**:
+	* Delete a location with the specified ID.
+5. **GET /location/search**:
+	* Get available locations from Open Meteo using the Geolocation API they provide
+
 
 ### Database Integration
 
-- Implement SQLAlchemy with a local Postgres database.
+- Implemented SQLAlchemy connection with a Postgres database.
-- Design a `Location` model with attributes including id, name, latitude, and longitude.
+- Implemented Models: 
-
+	- *Location` model with id, name, latitude, and longitude.*
-### API Endpoints
+	- *Users* model with 
-
+- Database Tables:
-- **Manage Locations:**
+	* locations
-  - `GET /locations`: Retrieve a list of all locations saved in the database, including their current weather conditions. This requires integrating with the [OpenMeteo API](https://open-meteo.com/) to fetch weather data based on latitude and longitude.
+		- For location storing
-  - `POST /locations`: Allow adding a new location by providing name, latitude, and longitude.
+	* users:
-  - `DELETE /locations/{id}`: Enable location deletion by ID.
+		- Prototype table for user information
-
+	* config:
-- **Weather Forecast:**
+		- Storing of user selected locations for the Dashboard
-  - `GET /forecast/{location_id}`: Provide a detailed 7-day weather forecast for a specified location. This endpoint will call the OpenMeteo API to fetch forecast data based on the location's latitude and longitude stored in the database.
 
 
-### API Integration
 
--   To fetch weather information, you are to use the [OpenMeteo API](https://open-meteo.com/). Given that this API requires latitude and longitude for location data, utilize [this predefined list of locations](https://gist.github.com/ofou/df09a6834a8421b4f376c875194915c9) as your hardcoded source.
+![Image](https://static1.argideli.com/weather-erd.png "Weather App ERD")
+
+### Installation
+To run the API locally, follow these steps:
+1. Clone the repository.
+2. Deploy and activate a venv
+3. Install the necessary dependencies by running `pip install -r requirements.txt`.
+4. Set up the database connection in the `config.py` file.
+5. Run the application by executing `uvicorn main:app --reload`.
+
+* Or you can use the Dockerfile for creating a docker image.
+	* *Also there is a public image at  git.argideli.com/argideli/weather_api:latest*

main.py

@@ -1,94 +1,143 @@
-from fastapi import FastAPI, status, HTTPException
-from models import Location
-from db_connector import database,engine,DB_REBUILD
 import utils
+from typing import Optional
+from pydantic import BaseModel
+from fastapi import FastAPI, status, HTTPException
+from db_connector import database,engine,DB_REBUILD
+
 
 if DB_REBUILD == 'True':
     database.metadata.drop_all(engine)
     utils.initialize_database()
 
+
+class Location(BaseModel):
+    id: int = None,
+    name: str = None,
+    country: str = None,
+    longitude: float = None,
+    latitude: float = None,
+    user: int = None,
+    
+class Users(BaseModel):
+    id: int = None,
+    name: str = None,
+    email: str = None,
+
+
+class Config(BaseModel):
+    id: int = None,
+    user_id: int = None,
+    location_id: int = None,
+
+
+
 app = FastAPI()
 
+def error_4xx_handler(return_data: dict) -> None:
+    """
+    Handles errors with status code 400.
+
+    Checks if the key 'error' exists in the `return_data` dictionary.
+    If it does, it raises an HTTPException with status code 400 and the value
+    of the 'error' key as the detail.
+
+    Args:
+        return_data (dict): Data returned from an API endpoint.
+
+    Raises:
+        HTTPException: If the 'error' key is present in `return_data`.
 
-def error_400_handler(return_data):
     """
-    A function that handles errors with status code 400. Takes return_data as input.
+
-    Checks if the key 'error' exists in the data dictionary. If it does, it raises an HTTPException
+    # Check if 'error' key exists in the return_data dictionary
-    with status code 400 and the value of the 'error' key as the detail.
-    """
-    
     if 'error' in return_data.keys():
+        # Raise HTTPException with status code 400 and the 'error' value as detail
         raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
+            status_code=status.HTTP_400_BAD_REQUEST,
-                detail={'error': return_data['error']}
+            detail={'error': return_data['error']}
         )
 
 
 @app.get("/")
 async def index_response():
+    """
+    A function that returns a status of "OK" when the root URL is accessed.
+    No parameters are passed, and it returns a dictionary with the status.
+    """
     return {"Status": "OK"}
 
 
 @app.get("/locations")
-async def get_location_weather(places: str):
+async def get_location_weather(
+    places: Optional[str] = 'all',
+    user: Optional[int] = None
+    ):
     """
     A function to retrieve weather data for specified locations.
-
+    
     Parameters:
     places (str): A string containing location identifiers separated by commas.
 
     Returns:
     dict: A dictionary containing weather data for the specified locations.
-    """
+    """    
-    
     #NOTE: Add option for fetching weather data for all locations, (debugging purposes)
-    if places != 'all':
+
-        places = [int(x) for x in list(places.split(","))]
+    if places == 'database_locations':
-    
+        result = utils.get_database_locations(user=None)
-    result = utils.retrieve_weather_data(places)
+    elif places == 'configured':
-    error_400_handler(result)
+        result = utils.get_database_locations(user)
+    else:
+        if places != 'all':
+            places = [int(x) for x in list(places.split(","))]
+        result = utils.retrieve_weather_data(places)
+        error_4xx_handler(result)
+        
     return result
 
 
 @app.get("/locations/{id}")
 async def get_weather_by_id(id: int):
     """
-    A function that retrieves weather data for a location by its ID.
+    Retrieves weather data for a specified location ID.
-    
+
     Parameters:
-    - id: an integer representing the ID of the location
+        id (int): The ID of the location to retrieve weather data for.
-    
+
     Returns:
-    - The weather data for the specified location
+        dict: A dictionary containing the retrieved weather data for the specified location.
     """
     
     result = utils.retrieve_weather_data([id])
-    error_400_handler(result)
+    error_4xx_handler(result)
     return result
     
 
 @app.post("/locations", status_code=status.HTTP_201_CREATED)
-async def add_location(
+async def add_location(loc: Location):
-    name: str = None,
-    longitude: float = None,
-    latitude: float = None,
-):
     """
     Add a new location to the database.
 
     Parameters:
-    - name (str): The name of the location.
+    - loc (Location): The location object to be added.
-    - longitude (float): The longitude coordinate of the location.
+
-    - latitude (float): The latitude coordinate of the location.
+    Raises:
+    - HTTPException: If the name exceeds 200 characters.
+
+    Returns:
+    None
     """
-    
+
-    if len(name.encode('utf-8')) > 200:
+    if len(loc.name.encode('utf-8')) > 200:
         raise HTTPException(
             status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
             detail={'name': 'Name cannot be longer than 200 characters'}
         )
     else:
-        utils.add_location(Location(name=name,longitude=longitude,latitude=latitude))
+        id = int(loc.id)
+        if loc.name!= 'existing':
+            id = utils.get_available_ids(1)[0]
+        utils.add_location({"id":id ,"name":loc.name,"country":loc.country,"longitude":loc.longitude,"latitude":loc.latitude, "user": loc.user}, no_commit=False)
 
 
 @app.delete("/locations/{id}")
@@ -98,9 +147,24 @@ async def delete_location(id: int):
 
     Parameters:
     id (int): The ID of the location to be deleted.
-
     Returns:
     None
     """
+
+    utils.delete_location(id)
+
+
+
+@app.get("/location/search")
+async def search_location(query: str):
+    """
+    A function to retrieve search results based on the provided query string.
     
-    utils.delete_location(id)
+    Parameters:
+    query (str): The search query string.
+    
+    Returns:
+    The search results based on the provided query.
+    """
+    result = utils.search_location(query)
+    return result

models.py

@@ -1,18 +1,67 @@
 from sqlalchemy.sql.expression import null
 from db_connector import database
-from sqlalchemy import Integer, Column, Float, VARCHAR
+from sqlalchemy import Integer, Column, Float, VARCHAR, ForeignKey
 
+#NOTE: Taumata longest place name on earth, 85 characters, 
+# also accounting for the extended ASCII characters,
+# varchar of 200 Bytes should be sufficient for every use 
+# case even with multiword names such as "The Big Apple"
+    
 class Location(database):
-    #NOTE: Taumata longest place name on earth, 85 characters, 
+    """
-    # also accounting for the extended ASCII characters,
+    Location model representing a physical location on Earth.
-    # varchar of 200 Bytes should be enough for every use 
+
-    # case even with multiword names such as "The Big Apple"
+    Attributes:
+    id (int): The unique identifier for each location.
+    name (str): The name of the location.
+    country (str): The country where the location is located.
+    latitude (float): The latitude coordinate of the location.
+    longitude (float): The longitude coordinate of the location.
+    """
+    
     __tablename__='locations'
-    id=Column(Integer,primary_key=True,autoincrement=True)
+    id=Column(Integer,primary_key=True)
     name=Column(VARCHAR(200),nullable=False)
+    country=Column(VARCHAR(100),nullable=False)
     latitude=Column(Float,nullable=False)
     longitude=Column(Float,nullable=False)
 
 
     def __repr__(self):
-        return "id={} name={} longitude={} latitude={}".format(self.id,self.name,self.longitude,self.latitude)
+        return "id={} name={} longitude={} latitude={}".format(self.id,self.name,self.longitude,self.latitude)
+    
+class Users(database):
+    """
+    Users model representing a registered user of the application.
+
+    Attributes:
+    id (int): The unique identifier for each user.
+    name (str): The name of the user.
+    email (str): The email address of the user.
+    """
+    
+    __tablename__='users'
+    id=Column(Integer,primary_key=True, autoincrement=True)
+    name=Column(VARCHAR(200),nullable=False)
+    email=Column(VARCHAR(100),nullable=False)
+
+    def __repr__(self):
+        return "id={} name={}".format(self.id,self.name)
+    
+class Config(database):
+    """
+    Config model representing a configuration for a user and location pair.
+
+    Attributes:
+    id (int): The unique identifier for each configuration.
+    user_id (int): The foreign key referencing the Users table.
+    location_id (int): The foreign key referencing the Locations table.
+    """
+    __tablename__='config'
+    id=Column(Integer,primary_key=True, autoincrement=True)
+    user_id=Column(Integer,ForeignKey(Users.id))
+    location_id=Column(Integer,ForeignKey(Location.id))
+    
+
+    def __repr__(self):
+        return "id={} user_id={} location_id={}".format(self.id,self.user_id,self.location_id)

utils.py

@@ -1,51 +1,114 @@
-from models import Location
+from models import Location, Users, Config
 from db_connector import database,engine,db_session
 import pandas as pd
 import openmeteo_requests
 import requests_cache
+import requests
 from retry_requests import retry
 
-
+    
 def initialize_database():
     """
     A function to initialize the database by checking table availability and creating it if it does not exist.
     """
+    db_session.add(Users(name="Argiris Deligiannidis",email="mai@argideli.com"))
     
     db_tables = ['locations']
     # check table availability and create it if it does not exist
     for tb in db_tables:
         if not engine.dialect.has_table(engine.connect(), tb):
-            print("Creating table: {}\n".format(tb))
+            print("\t*** Creating tables ***")
             database.metadata.create_all(engine)
             
             table_data = pd.read_csv ('./table_data/locations.csv', index_col=None, header=0)
-            for i in range(len(table_data)):
+            for idx in range(len(table_data)):
-                add_location(Location(name=table_data.loc[i, "Capital City"],
+                location = {
-                                    latitude=float(table_data.loc[i, "Latitude"]),
+                    'id': idx+1,
-                                    longitude=float(table_data.loc[i, "Longitude"]),
+                    'name': table_data.loc[idx, "Capital City"],
-                                    ),
+                    'country': table_data.loc[idx, "Country"],
-                                    no_commit=True
+                    'latitude': float(table_data.loc[idx, "Latitude"]),
-                            )
+                    'longitude': float(table_data.loc[idx, "Longitude"]),
+                    'user': 'bootstrap',
+                }
+                
+                add_location(location, no_commit=True)
             db_session.commit()
-            
+
-def add_location(location:Location , no_commit=False):
+def get_database_locations(user=None):
+    """
+    Retrieve locations from the database for a specified user, or all locations if no user is specified.
+    """
+    if user is not None:
+        locations = db_session.query(Config).filter(Config.user_id == user).all()
+    else:
+        locations = db_session.query(Location).all()
+        
+    
+    return locations
+
+def get_max_id():
+    """
+    A function to retrieve the maximum ID from the Location table in the database.
+    """
+    return max([id[0] for id in db_session.query(Location.id).all()])
+
+def get_available_ids(id_num):
+    """
+    Function to generate a list of available IDs based on existing IDs in the database.
+    Parameters:
+        id_num (int): The number of IDs to generate.
+    Returns:
+        List[int]: List of available IDs.
+    """
+    db_ids = [id[0] for id in db_session.query(Location.id).all()]
+    avail_ids = [loc for loc in range(max(db_ids)+1) if loc not in db_ids and loc != 0]
+
+    for i in range(id_num-len(avail_ids)):
+        if avail_ids != []:
+            avail_ids.append(max(avail_ids)+1)
+        else:
+            avail_ids.append(max(db_ids)+1)
+
+    return avail_ids
+
+def add_location(location, no_commit=False):
     """
     A function that adds a location to the database session.
 
     Parameters:
     location (Location): The location object to be added to the database session.
     no_commit (bool): Flag indicating whether to commit the transaction immediately.
-
-    Returns:
-    None
     """
+    if  location["name"] != 'existing':
+        db_session.add(Location(id=location["id"],
+                                        name=location["name"],
+                                        country=location["country"],
+                                        latitude=location["latitude"],
+                                        longitude=location["longitude"],
+                                        )
+                            )
+        if not no_commit:
+            db_session.commit()
     
-    db_session.add(location)
+    if location["user"] != 'bootstrap':
-    #print("Adding location: {}".format(location))
+        db_session.add(Config(user_id=location["user"],location_id=location["id"]))
-    if not no_commit:
+        no_commit == False
-        db_session.commit()
+
-        
+
+def config_disable_location(id, user):
+    """
+    A function that disables a location configuration based on the provided ID and user.
+    
+    Parameters:
+        id (int): The ID of the location to be disabled.
+        user (int): The user ID associated with the location configuration.
         
+    Returns:
+        None
+    """
+    db_session.add(Config(user_id=user,location_id=id))
+    db_session.commit()
+
 def delete_location(id):
     """
     Deletes a location from the database based on the provided ID.
@@ -56,24 +119,27 @@ def delete_location(id):
     Returns:
         dict: A dictionary with the key "id" indicating that the location was successfully deleted.
     """
-    
+    db_session.query(Config).filter(Config.location_id == id).delete()
+    db_session.commit()
     db_session.query(Location).filter(Location.id == id).delete()
     db_session.commit()
+    
     return {"id": "Deleted"}
 
 
 def chunkify_data(data, chunk_size):
     """
-    A function that chunks the input data into smaller pieces of the specified chunk size.
+    A function to split data into chunks of a specified size for processing.
-    
+
     Parameters:
-    - data: the input data to be chunked
+    - data: The input data to be chunked.
-    - chunk_size: the size of each chunk
+    - chunk_size: The size of each chunk to split the data into.
-    
+
     Returns:
-    - A generator that yields chunks of the input data
+    - A generator that yields chunks of the data based on the specified chunk size.
     """
-    
+    #NOTE bulk operation: Open Weather api has an upper limit of ~= 180 parameters for a request per second
+    # so we will split the requests into chunks of 100 parameters
     for i in range(0, len(data), chunk_size):  
         yield data[i:i + chunk_size] 
 
@@ -91,20 +157,17 @@ def retrieve_weather_data(location_id=None):
     - If location_id is 'all', weather data for all locations is returned in a dictionary.
     - Otherwise, weather data for the specified location IDs is returned in a dictionary.
     """
-    print('xaz')
+    max_id = get_max_id()
-    max_id = db_session.query(Location).count()
+
-    
     #NOTE: Disable location check if location_id is 'all' (returns all locations), debugging purposes
     if location_id != 'all':
         loc_check = {}
         for loc_id in location_id:
             if loc_id > max_id:
                 loc_check[loc_id] = "The location does not exist"
-        print(len(loc_check))
         if len(loc_check) > 0:
             return {'error': loc_check}
 
-    print('az')
     weather_data = {}
     
     #NOTE: Get weather data for all locations if location_id is 'all' (debugging purposes), otherwise get weather data for specified locations
@@ -112,12 +175,15 @@ def retrieve_weather_data(location_id=None):
         locations = list(chunkify_data(db_session.query(Location).all(),100))
     else:
         locations = list(chunkify_data(db_session.query(Location).filter(Location.id.in_(location_id)).all(),100))
-        print(locations)
+        
+    
     for chunk in locations:
-        coordinates = [[loc.id, loc.latitude, loc.longitude, loc.name] for loc in chunk]
+        coordinates = []
+        for i in range(len(chunk)):
+            ids = get_available_ids(len(chunk))
+            coordinates.append([chunk[i].id, chunk[i].latitude, chunk[i].longitude, chunk[i].name])
         weather_data.update(get_openmeteo_data(coordinates))
-    #print(weather_data)
+
-    #exit(0)
     return weather_data
             
 def get_openmeteo_data(coordinates):
@@ -151,7 +217,8 @@ def get_openmeteo_data(coordinates):
     #     "shortwave_radiation_sum", "et0_fao_evapotranspiration"
     # ]
     
-    data_names = ["temperature_2m_max", "temperature_2m_min", "precipitation_sum"]
+    current_names = ["weather_code", "temperature_2m", "rain", "precipitation", "showers"]
+    data_names = ["weather_code", "temperature_2m_max", "temperature_2m_min", "rain_sum"]
 
     #NOTE: OpenMeteo API has a limit of 10000 requests per day,
     # so in a production environment it would be wise to change to 
@@ -160,6 +227,7 @@ def get_openmeteo_data(coordinates):
     params = {
         "latitude": latitude,
         "longitude": longitude,
+        "current": current_names,
         "daily": data_names
     }
 
@@ -169,13 +237,22 @@ def get_openmeteo_data(coordinates):
         location = coordinates[idx-1][3]
         idx = coordinates[idx-1][0]
         
-        daily = response.Daily()
+        current = response.Current()
         
-        daily_data = { variable: daily.Variables(i).ValuesAsNumpy().tolist()
+        #set rain to max of the available openmeteo result variables
+        max_rain = max([float(current.Variables(2).Value()), 
+                    float(current.Variables(3).Value()), 
+                    float(current.Variables(3).Value())])
+        
+        current_data = {"weather_code": current.Variables(0).Value(), "temperature_2m": current.Variables(1).Value(), "rain": max_rain}
+
+        daily = response.Daily()
+        daily_data = {}
+        daily_data.update({ variable: daily.Variables(i).ValuesAsNumpy().tolist()
                     if variable not in ["sunrise", "sunset"]
                     else daily.Variables(i).ValuesAsNumpy()
                     for i, variable in enumerate(data_names)
-                    }
+                    })
         
         dates = pd.date_range(start = pd.to_datetime(daily.Time(), unit = "s", utc = True),
                             end = pd.to_datetime(daily.TimeEnd(), unit = "s", utc = True),
@@ -183,10 +260,51 @@ def get_openmeteo_data(coordinates):
                             inclusive = "left").tolist()
         
         daily_data["date"] = [date.strftime("%d/%m/%Y") for date in dates]
-        daily_data["location_name"] = location
-        daily_data["resp_coordinates"] = [response.Latitude(), response.Longitude()]
         
-        data_dict[idx] = daily_data
+        location_data = {
+            "id": idx,
+            "name": location,
+            "coordinates": [response.Latitude(), response.Longitude()]
+        }
+
+        data_dict[idx] = {
+            "id": idx,
+            "data": {
+                "location": location_data,
+                "current": current_data,
+                "daily": daily_data
+            }
+        }
 
     return data_dict
 
+def search_location(query):
+    """
+    Retrieve location data based on the provided query string.
+    
+    Parameters:
+    query (str): The query string used to search for locations.
+    
+    Returns:
+    A dictionary containing location data with an index as the key and location information as the value.
+    """
+    
+    URL="https://geocoding-api.open-meteo.com/v1/search"
+    PARAMS = {
+        "name": query,
+        "count": 10,
+        "language": 'en',
+        "format": 'json'
+    }
+
+    response = requests.get(url = URL, params = PARAMS)
+    data_dict={}
+    try:
+        for idx,d in enumerate(response.json()['results']):
+            data_dict[idx] = d
+            data_dict[idx].update({'selected': False})
+    except KeyError:
+        data_dict[0] = {'result': 'Error'}
+
+    return data_dict
+