From 902bd905afd2d829c009887a5c45bfbe0bac12e9 Mon Sep 17 00:00:00 2001 From: Argiris Deligiannidis Date: Sun, 14 Apr 2024 23:00:07 +0300 Subject: [PATCH] Update models, functions and Readme --- README.md | 54 ++++++++++---- main.py | 142 ++++++++++++++++++++++++++----------- models.py | 63 +++++++++++++++-- utils.py | 206 ++++++++++++++++++++++++++++++++++++++++++------------ 4 files changed, 360 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 95d02b7..28b2290 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,47 @@ ## weather_api -### Datacose Weather API +### Overview +This API serves as a backend system for managing the location of the Open Meteo Weather dashboard. It includes functionalities for creating, updating, deleting and searching available locations. + + +### Endpoints +1. **GET /locations**: + * Retrieve a list of all the locations stored in the database. + * The locations configured by the user +2. **GET /locations/{id}**: + * Retriev weather data for the specified ID. +3. **POST /locations**: Add a new location to the database. +4. **DELETE /locations/{id}**: + * Delete a location with the specified ID. +5. **GET /location/search**: + * Get available locations from Open Meteo using the Geolocation API they provide + ### Database Integration -- Implement SQLAlchemy with a local Postgres database. -- Design a `Location` model with attributes including id, name, latitude, and longitude. - -### API Endpoints - -- **Manage Locations:** - - `GET /locations`: Retrieve a list of all locations saved in the database, including their current weather conditions. This requires integrating with the [OpenMeteo API](https://open-meteo.com/) to fetch weather data based on latitude and longitude. - - `POST /locations`: Allow adding a new location by providing name, latitude, and longitude. - - `DELETE /locations/{id}`: Enable location deletion by ID. - -- **Weather Forecast:** - - `GET /forecast/{location_id}`: Provide a detailed 7-day weather forecast for a specified location. This endpoint will call the OpenMeteo API to fetch forecast data based on the location's latitude and longitude stored in the database. +- Implemented SQLAlchemy connection with a Postgres database. +- Implemented Models: + - *Location` model with id, name, latitude, and longitude.* + - *Users* model with +- Database Tables: + * locations + - For location storing + * users: + - Prototype table for user information + * config: + - Storing of user selected locations for the Dashboard -### API Integration -- To fetch weather information, you are to use the [OpenMeteo API](https://open-meteo.com/). Given that this API requires latitude and longitude for location data, utilize [this predefined list of locations](https://gist.github.com/ofou/df09a6834a8421b4f376c875194915c9) as your hardcoded source. +![Image](https://static1.argideli.com/weather-erd.png "Weather App ERD") + +### Installation +To run the API locally, follow these steps: +1. Clone the repository. +2. Deploy and activate a venv +3. Install the necessary dependencies by running `pip install -r requirements.txt`. +4. Set up the database connection in the `config.py` file. +5. Run the application by executing `uvicorn main:app --reload`. + +* Or you can use the Dockerfile for creating a docker image. + * *Also there is a public image at git.argideli.com/argideli/weather_api:latest* diff --git a/main.py b/main.py index 88b6e46..078a5a3 100644 --- a/main.py +++ b/main.py @@ -1,94 +1,143 @@ -from fastapi import FastAPI, status, HTTPException -from models import Location -from db_connector import database,engine,DB_REBUILD import utils +from typing import Optional +from pydantic import BaseModel +from fastapi import FastAPI, status, HTTPException +from db_connector import database,engine,DB_REBUILD + if DB_REBUILD == 'True': database.metadata.drop_all(engine) utils.initialize_database() + +class Location(BaseModel): + id: int = None, + name: str = None, + country: str = None, + longitude: float = None, + latitude: float = None, + user: int = None, + +class Users(BaseModel): + id: int = None, + name: str = None, + email: str = None, + + +class Config(BaseModel): + id: int = None, + user_id: int = None, + location_id: int = None, + + + app = FastAPI() +def error_4xx_handler(return_data: dict) -> None: + """ + Handles errors with status code 400. + + Checks if the key 'error' exists in the `return_data` dictionary. + If it does, it raises an HTTPException with status code 400 and the value + of the 'error' key as the detail. + + Args: + return_data (dict): Data returned from an API endpoint. + + Raises: + HTTPException: If the 'error' key is present in `return_data`. -def error_400_handler(return_data): """ - A function that handles errors with status code 400. Takes return_data as input. - Checks if the key 'error' exists in the data dictionary. If it does, it raises an HTTPException - with status code 400 and the value of the 'error' key as the detail. - """ - + + # Check if 'error' key exists in the return_data dictionary if 'error' in return_data.keys(): + # Raise HTTPException with status code 400 and the 'error' value as detail raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail={'error': return_data['error']} + status_code=status.HTTP_400_BAD_REQUEST, + detail={'error': return_data['error']} ) @app.get("/") async def index_response(): + """ + A function that returns a status of "OK" when the root URL is accessed. + No parameters are passed, and it returns a dictionary with the status. + """ return {"Status": "OK"} @app.get("/locations") -async def get_location_weather(places: str): +async def get_location_weather( + places: Optional[str] = 'all', + user: Optional[int] = None + ): """ A function to retrieve weather data for specified locations. - + Parameters: places (str): A string containing location identifiers separated by commas. Returns: dict: A dictionary containing weather data for the specified locations. - """ - + """ #NOTE: Add option for fetching weather data for all locations, (debugging purposes) - if places != 'all': - places = [int(x) for x in list(places.split(","))] - - result = utils.retrieve_weather_data(places) - error_400_handler(result) + + if places == 'database_locations': + result = utils.get_database_locations(user=None) + elif places == 'configured': + result = utils.get_database_locations(user) + else: + if places != 'all': + places = [int(x) for x in list(places.split(","))] + result = utils.retrieve_weather_data(places) + error_4xx_handler(result) + return result @app.get("/locations/{id}") async def get_weather_by_id(id: int): """ - A function that retrieves weather data for a location by its ID. - + Retrieves weather data for a specified location ID. + Parameters: - - id: an integer representing the ID of the location - + id (int): The ID of the location to retrieve weather data for. + Returns: - - The weather data for the specified location + dict: A dictionary containing the retrieved weather data for the specified location. """ result = utils.retrieve_weather_data([id]) - error_400_handler(result) + error_4xx_handler(result) return result @app.post("/locations", status_code=status.HTTP_201_CREATED) -async def add_location( - name: str = None, - longitude: float = None, - latitude: float = None, -): +async def add_location(loc: Location): """ Add a new location to the database. Parameters: - - name (str): The name of the location. - - longitude (float): The longitude coordinate of the location. - - latitude (float): The latitude coordinate of the location. + - loc (Location): The location object to be added. + + Raises: + - HTTPException: If the name exceeds 200 characters. + + Returns: + None """ - - if len(name.encode('utf-8')) > 200: + + if len(loc.name.encode('utf-8')) > 200: raise HTTPException( status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail={'name': 'Name cannot be longer than 200 characters'} ) else: - utils.add_location(Location(name=name,longitude=longitude,latitude=latitude)) + id = int(loc.id) + if loc.name!= 'existing': + id = utils.get_available_ids(1)[0] + utils.add_location({"id":id ,"name":loc.name,"country":loc.country,"longitude":loc.longitude,"latitude":loc.latitude, "user": loc.user}, no_commit=False) @app.delete("/locations/{id}") @@ -98,9 +147,24 @@ async def delete_location(id: int): Parameters: id (int): The ID of the location to be deleted. - Returns: None """ + + utils.delete_location(id) + + + +@app.get("/location/search") +async def search_location(query: str): + """ + A function to retrieve search results based on the provided query string. - utils.delete_location(id) \ No newline at end of file + Parameters: + query (str): The search query string. + + Returns: + The search results based on the provided query. + """ + result = utils.search_location(query) + return result diff --git a/models.py b/models.py index 633249a..d5bd349 100644 --- a/models.py +++ b/models.py @@ -1,18 +1,67 @@ from sqlalchemy.sql.expression import null from db_connector import database -from sqlalchemy import Integer, Column, Float, VARCHAR +from sqlalchemy import Integer, Column, Float, VARCHAR, ForeignKey +#NOTE: Taumata longest place name on earth, 85 characters, +# also accounting for the extended ASCII characters, +# varchar of 200 Bytes should be sufficient for every use +# case even with multiword names such as "The Big Apple" + class Location(database): - #NOTE: Taumata longest place name on earth, 85 characters, - # also accounting for the extended ASCII characters, - # varchar of 200 Bytes should be enough for every use - # case even with multiword names such as "The Big Apple" + """ + Location model representing a physical location on Earth. + + Attributes: + id (int): The unique identifier for each location. + name (str): The name of the location. + country (str): The country where the location is located. + latitude (float): The latitude coordinate of the location. + longitude (float): The longitude coordinate of the location. + """ + __tablename__='locations' - id=Column(Integer,primary_key=True,autoincrement=True) + id=Column(Integer,primary_key=True) name=Column(VARCHAR(200),nullable=False) + country=Column(VARCHAR(100),nullable=False) latitude=Column(Float,nullable=False) longitude=Column(Float,nullable=False) def __repr__(self): - return "id={} name={} longitude={} latitude={}".format(self.id,self.name,self.longitude,self.latitude) \ No newline at end of file + return "id={} name={} longitude={} latitude={}".format(self.id,self.name,self.longitude,self.latitude) + +class Users(database): + """ + Users model representing a registered user of the application. + + Attributes: + id (int): The unique identifier for each user. + name (str): The name of the user. + email (str): The email address of the user. + """ + + __tablename__='users' + id=Column(Integer,primary_key=True, autoincrement=True) + name=Column(VARCHAR(200),nullable=False) + email=Column(VARCHAR(100),nullable=False) + + def __repr__(self): + return "id={} name={}".format(self.id,self.name) + +class Config(database): + """ + Config model representing a configuration for a user and location pair. + + Attributes: + id (int): The unique identifier for each configuration. + user_id (int): The foreign key referencing the Users table. + location_id (int): The foreign key referencing the Locations table. + """ + __tablename__='config' + id=Column(Integer,primary_key=True, autoincrement=True) + user_id=Column(Integer,ForeignKey(Users.id)) + location_id=Column(Integer,ForeignKey(Location.id)) + + + def __repr__(self): + return "id={} user_id={} location_id={}".format(self.id,self.user_id,self.location_id) \ No newline at end of file diff --git a/utils.py b/utils.py index 4f7e590..0bc3267 100644 --- a/utils.py +++ b/utils.py @@ -1,51 +1,114 @@ -from models import Location +from models import Location, Users, Config from db_connector import database,engine,db_session import pandas as pd import openmeteo_requests import requests_cache +import requests from retry_requests import retry - + def initialize_database(): """ A function to initialize the database by checking table availability and creating it if it does not exist. """ + db_session.add(Users(name="Argiris Deligiannidis",email="mai@argideli.com")) db_tables = ['locations'] # check table availability and create it if it does not exist for tb in db_tables: if not engine.dialect.has_table(engine.connect(), tb): - print("Creating table: {}\n".format(tb)) + print("\t*** Creating tables ***") database.metadata.create_all(engine) table_data = pd.read_csv ('./table_data/locations.csv', index_col=None, header=0) - for i in range(len(table_data)): - add_location(Location(name=table_data.loc[i, "Capital City"], - latitude=float(table_data.loc[i, "Latitude"]), - longitude=float(table_data.loc[i, "Longitude"]), - ), - no_commit=True - ) + for idx in range(len(table_data)): + location = { + 'id': idx+1, + 'name': table_data.loc[idx, "Capital City"], + 'country': table_data.loc[idx, "Country"], + 'latitude': float(table_data.loc[idx, "Latitude"]), + 'longitude': float(table_data.loc[idx, "Longitude"]), + 'user': 'bootstrap', + } + + add_location(location, no_commit=True) db_session.commit() - -def add_location(location:Location , no_commit=False): + +def get_database_locations(user=None): + """ + Retrieve locations from the database for a specified user, or all locations if no user is specified. + """ + if user is not None: + locations = db_session.query(Config).filter(Config.user_id == user).all() + else: + locations = db_session.query(Location).all() + + + return locations + +def get_max_id(): + """ + A function to retrieve the maximum ID from the Location table in the database. + """ + return max([id[0] for id in db_session.query(Location.id).all()]) + +def get_available_ids(id_num): + """ + Function to generate a list of available IDs based on existing IDs in the database. + Parameters: + id_num (int): The number of IDs to generate. + Returns: + List[int]: List of available IDs. + """ + db_ids = [id[0] for id in db_session.query(Location.id).all()] + avail_ids = [loc for loc in range(max(db_ids)+1) if loc not in db_ids and loc != 0] + + for i in range(id_num-len(avail_ids)): + if avail_ids != []: + avail_ids.append(max(avail_ids)+1) + else: + avail_ids.append(max(db_ids)+1) + + return avail_ids + +def add_location(location, no_commit=False): """ A function that adds a location to the database session. Parameters: location (Location): The location object to be added to the database session. no_commit (bool): Flag indicating whether to commit the transaction immediately. - - Returns: - None """ + if location["name"] != 'existing': + db_session.add(Location(id=location["id"], + name=location["name"], + country=location["country"], + latitude=location["latitude"], + longitude=location["longitude"], + ) + ) + if not no_commit: + db_session.commit() - db_session.add(location) - #print("Adding location: {}".format(location)) - if not no_commit: - db_session.commit() - + if location["user"] != 'bootstrap': + db_session.add(Config(user_id=location["user"],location_id=location["id"])) + no_commit == False + + +def config_disable_location(id, user): + """ + A function that disables a location configuration based on the provided ID and user. + + Parameters: + id (int): The ID of the location to be disabled. + user (int): The user ID associated with the location configuration. + Returns: + None + """ + db_session.add(Config(user_id=user,location_id=id)) + db_session.commit() + def delete_location(id): """ Deletes a location from the database based on the provided ID. @@ -56,24 +119,27 @@ def delete_location(id): Returns: dict: A dictionary with the key "id" indicating that the location was successfully deleted. """ - + db_session.query(Config).filter(Config.location_id == id).delete() + db_session.commit() db_session.query(Location).filter(Location.id == id).delete() db_session.commit() + return {"id": "Deleted"} def chunkify_data(data, chunk_size): """ - A function that chunks the input data into smaller pieces of the specified chunk size. - + A function to split data into chunks of a specified size for processing. + Parameters: - - data: the input data to be chunked - - chunk_size: the size of each chunk - + - data: The input data to be chunked. + - chunk_size: The size of each chunk to split the data into. + Returns: - - A generator that yields chunks of the input data + - A generator that yields chunks of the data based on the specified chunk size. """ - + #NOTE bulk operation: Open Weather api has an upper limit of ~= 180 parameters for a request per second + # so we will split the requests into chunks of 100 parameters for i in range(0, len(data), chunk_size): yield data[i:i + chunk_size] @@ -91,20 +157,17 @@ def retrieve_weather_data(location_id=None): - If location_id is 'all', weather data for all locations is returned in a dictionary. - Otherwise, weather data for the specified location IDs is returned in a dictionary. """ - print('xaz') - max_id = db_session.query(Location).count() - + max_id = get_max_id() + #NOTE: Disable location check if location_id is 'all' (returns all locations), debugging purposes if location_id != 'all': loc_check = {} for loc_id in location_id: if loc_id > max_id: loc_check[loc_id] = "The location does not exist" - print(len(loc_check)) if len(loc_check) > 0: return {'error': loc_check} - print('az') weather_data = {} #NOTE: Get weather data for all locations if location_id is 'all' (debugging purposes), otherwise get weather data for specified locations @@ -112,12 +175,15 @@ def retrieve_weather_data(location_id=None): locations = list(chunkify_data(db_session.query(Location).all(),100)) else: locations = list(chunkify_data(db_session.query(Location).filter(Location.id.in_(location_id)).all(),100)) - print(locations) + + for chunk in locations: - coordinates = [[loc.id, loc.latitude, loc.longitude, loc.name] for loc in chunk] + coordinates = [] + for i in range(len(chunk)): + ids = get_available_ids(len(chunk)) + coordinates.append([chunk[i].id, chunk[i].latitude, chunk[i].longitude, chunk[i].name]) weather_data.update(get_openmeteo_data(coordinates)) - #print(weather_data) - #exit(0) + return weather_data def get_openmeteo_data(coordinates): @@ -151,7 +217,8 @@ def get_openmeteo_data(coordinates): # "shortwave_radiation_sum", "et0_fao_evapotranspiration" # ] - data_names = ["temperature_2m_max", "temperature_2m_min", "precipitation_sum"] + current_names = ["weather_code", "temperature_2m", "rain", "precipitation", "showers"] + data_names = ["weather_code", "temperature_2m_max", "temperature_2m_min", "rain_sum"] #NOTE: OpenMeteo API has a limit of 10000 requests per day, # so in a production environment it would be wise to change to @@ -160,6 +227,7 @@ def get_openmeteo_data(coordinates): params = { "latitude": latitude, "longitude": longitude, + "current": current_names, "daily": data_names } @@ -169,13 +237,22 @@ def get_openmeteo_data(coordinates): location = coordinates[idx-1][3] idx = coordinates[idx-1][0] - daily = response.Daily() + current = response.Current() - daily_data = { variable: daily.Variables(i).ValuesAsNumpy().tolist() + #set rain to max of the available openmeteo result variables + max_rain = max([float(current.Variables(2).Value()), + float(current.Variables(3).Value()), + float(current.Variables(3).Value())]) + + current_data = {"weather_code": current.Variables(0).Value(), "temperature_2m": current.Variables(1).Value(), "rain": max_rain} + + daily = response.Daily() + daily_data = {} + daily_data.update({ variable: daily.Variables(i).ValuesAsNumpy().tolist() if variable not in ["sunrise", "sunset"] else daily.Variables(i).ValuesAsNumpy() for i, variable in enumerate(data_names) - } + }) dates = pd.date_range(start = pd.to_datetime(daily.Time(), unit = "s", utc = True), end = pd.to_datetime(daily.TimeEnd(), unit = "s", utc = True), @@ -183,10 +260,51 @@ def get_openmeteo_data(coordinates): inclusive = "left").tolist() daily_data["date"] = [date.strftime("%d/%m/%Y") for date in dates] - daily_data["location_name"] = location - daily_data["resp_coordinates"] = [response.Latitude(), response.Longitude()] - data_dict[idx] = daily_data + location_data = { + "id": idx, + "name": location, + "coordinates": [response.Latitude(), response.Longitude()] + } + + data_dict[idx] = { + "id": idx, + "data": { + "location": location_data, + "current": current_data, + "daily": daily_data + } + } return data_dict +def search_location(query): + """ + Retrieve location data based on the provided query string. + + Parameters: + query (str): The query string used to search for locations. + + Returns: + A dictionary containing location data with an index as the key and location information as the value. + """ + + URL="https://geocoding-api.open-meteo.com/v1/search" + PARAMS = { + "name": query, + "count": 10, + "language": 'en', + "format": 'json' + } + + response = requests.get(url = URL, params = PARAMS) + data_dict={} + try: + for idx,d in enumerate(response.json()['results']): + data_dict[idx] = d + data_dict[idx].update({'selected': False}) + except KeyError: + data_dict[0] = {'result': 'Error'} + + return data_dict + \ No newline at end of file