diff --git a/.gitignore b/.gitignore index 0b8d49f9b..bb94808bf 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,17 @@ xcuserdata/ *.pte *.model Package.resolved + +#embeddings +**/faiss_index.index +**/faiss_index.index.json +**/chroma + + +# DocQA +/examples/DocQA/data/input +/examples/DocQA/data/output +**/.gradio +**/RAG_service.json +examples/DocQA/example_data/chroma.sqlite3 +examples/DocQA/example_data/**/*.bin diff --git a/examples/DocQA/README.md b/examples/DocQA/README.md new file mode 100644 index 000000000..7b8cfee31 --- /dev/null +++ b/examples/DocQA/README.md @@ -0,0 +1,51 @@ +## DocQA + +This is an end-to-end Retrieval Augmented Generation (RAG) App leveraging llama-stack that handles the logic for ingesting documents, storing them in a vector database and providing an inference interface. + +We share the details of how to run first and then an outline of how it works: + +### Prerequisite: + +Install docker: Check [this doc for Mac](https://docs.docker.com/desktop/setup/install/mac-install/), [this doc for Windows](https://docs.docker.com/desktop/setup/install/windows-install/) and this [instruction for Linux](https://docs.docker.com/engine/install/). + +For Mac and Windows users, you need to start the Docker app manually after installation. + +### How to run the pipeline: + +![RAG_workflow](./data/assets/DocQA.png) + +The above is the workflow diagram for this RAG app. To run the app, please read the following instructions: + +1. Copy the template configuration file `docqa_env_template` to create your own `docqv_env` inside the docker folder: + +```bash +cd docker +cp docqa_env_template docqv_env +``` + +2. Then update `model_name` and `document_path` accordingly in your `docqv_env`, for example: + +``` +DOC_PATH=/path/to/your/llama-stack-apps/examples/DocQA/example_data +MODEL_NAME=llama3.2:1b-instruct-fp16 +HOST=localhost +LLAMA_STACK_PORT=5000 +CHROMA_PORT=6000 +GRADIO_SERVER_PORT=7860 +USE_GPU_FOR_DOC_INGESTION=false +``` + +3. In the `docker` folder, run following code: + +```bash +bash run_RAG.sh +``` + +4. Once the service is ready, open the link http://localhost:7861/ in your browser to chat with your documents. + +### Overview of how the RAG app works: + +1. We use [docling](https://github.com/DS4SD/docling) framework for handling multiple file input formats (PDF, PPTX, DOCX) +2. If you are using a GPU, we have an option to use `Llama-3.2-11B-Vision` to caption images in the documents. On a CPU-only machine this step is skipped. +3. Once ingested, we use a llama-stack distribution running chroma-db and `Llama-3.2-3B-Instruct` to ingest chunks into a memory_bank +4. Once the vectordb is created, we then use llama-stack with the `Llama-3.2-3B-Instruct` to chat with the model. diff --git a/examples/DocQA/app.py b/examples/DocQA/app.py new file mode 100644 index 000000000..3eff8fa52 --- /dev/null +++ b/examples/DocQA/app.py @@ -0,0 +1,301 @@ +import asyncio +import json +import os +import re +import uuid +from queue import Queue +from threading import Thread +from typing import AsyncGenerator, Generator, List, Optional + +import chromadb +import gradio as gr +import requests +from chromadb.utils import embedding_functions +from dotenv import load_dotenv +from llama_stack_client import LlamaStackClient +from llama_stack_client.lib.agents.agent import Agent +from llama_stack_client.lib.agents.event_logger import EventLogger +from llama_stack_client.types.agent_create_params import AgentConfig +from llama_stack_client.types.memory_insert_params import Document + + +# Load environment variables +load_dotenv() + +HOST = os.getenv("HOST", "localhost") +LLAMA_STACK_PORT = int(os.getenv("LLAMA_STACK_PORT", "5000")) +GRADIO_SERVER_PORT = int(os.getenv("GRADIO_SERVER_PORT", "7861")) +USE_GPU_FOR_DOC_INGESTION = os.getenv("USE_GPU_FOR_DOC_INGESTION", False) +MODEL_NAME = os.getenv("MODEL_NAME", "meta-llama/Llama-3.2-1B-Instruct") +# if USE_GPU_FOR_DOC_INGESTION, then the documents will be processed to output folder +DOCS_DIR = "/root/rag_data/output" if USE_GPU_FOR_DOC_INGESTION else "/root/rag_data/" + +CUSTOM_CSS = """ +.context-block { + font-size: 0.8em; + border-left: 3px solid #e9ecef; + margin: 0.5em 0; + padding: 0.5em 1em; + opacity: 0.85; +} + +.context-title { + font-size: 0.8em; + color: #9ca3af; + font-weight: 400; + display: flex; + align-items: center; + gap: 0.5em; + margin-bottom: 0.3em; + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.context-title::before { + content: "📄"; + font-size: 1em; + opacity: 0.7; +} + +.context-content { + color: #6b7280; + line-height: 1.4; + font-weight: 400; +} + +.inference-response { + font-size: 1em; + color: #111827; + line-height: 1.5; + margin-top: 1em; +} +""" + + +class LlamaChatInterface: + def __init__(self, host: str, port: int, docs_dir: str): + self.host = host + self.port = port + self.docs_dir = docs_dir + self.client = LlamaStackClient(base_url=f"http://{host}:{port}") + self.agent = None + self.session_id = None + self.memory_bank_id = "docqa_bank" + + async def initialize_system(self): + """Initialize the entire system including memory bank and agent.""" + await self.setup_memory_bank() + await self.initialize_agent() + + async def setup_memory_bank(self): + """Set up the memory bank if it doesn't exist.""" + providers = self.client.providers.list() + provider_id = providers["memory"][0].provider_id + memory_banks = self.client.memory_banks.list() + print(f"Memory banks: {memory_banks}") + + # Check if memory bank exists by identifier + if any(bank.identifier == self.memory_bank_id for bank in memory_banks): + print(f"Memory bank '{self.memory_bank_id}' exists.") + else: + print(f"Memory bank '{self.memory_bank_id}' does not exist. Creating...") + self.client.memory_banks.register( + memory_bank_id=self.memory_bank_id, + params={ + "embedding_model": "all-MiniLM-L6-v2", + "chunk_size_in_tokens": 100, + "overlap_size_in_tokens": 10, + }, + provider_id=provider_id, + ) + await self.load_documents() + print(f"Memory bank registered.") + + async def load_documents(self): + """Load documents from the specified directory into memory bank.""" + documents = [] + for filename in os.listdir(self.docs_dir): + if filename.endswith((".txt", ".md")): + file_path = os.path.join(self.docs_dir, filename) + with open(file_path, "r", encoding="utf-8") as file: + content = file.read() + document = Document( + document_id=filename, + content=content, + mime_type="text/plain", + metadata={"filename": filename}, + ) + documents.append(document) + + if documents: + self.client.memory.insert( + bank_id=self.memory_bank_id, + documents=documents, + ) + print(f"Loaded {len(documents)} documents from {self.docs_dir}") + + async def initialize_agent(self): + """Initialize the agent with model registration and configuration.""" + + if "1b" in MODEL_NAME: + model_name = "Llama3.2-1B-Instruct" + elif "3b" in MODEL_NAME: + model_name = "Llama3.2-3B-Instruct" + elif "8b" in MODEL_NAME: + model_name = "Llama3.1-8B-Instruct" + else: + model_name = MODEL_NAME + + agent_config = AgentConfig( + model=model_name, + instructions="You are a helpful assistant that can answer questions based on provided documents. Return your answer short and concise, less than 50 words.", + sampling_params={"strategy": "greedy", "temperature": 1.0, "top_p": 0.9}, + tools=[ + { + "type": "memory", + "memory_bank_configs": [ + {"bank_id": self.memory_bank_id, "type": "vector"} + ], + "max_tokens_in_context": 300, + "max_chunks": 5, + } + ], + tool_choice="auto", + tool_prompt_format="json", + enable_session_persistence=True, + ) + self.agent = Agent(self.client, agent_config) + self.session_id = self.agent.create_session(f"session-{uuid.uuid4()}") + + def chat_stream( + self, message: str, history: List[List[str]] + ) -> Generator[List[List[str]], None, None]: + """Stream chat responses token by token with proper history handling.""" + + history = history or [] + history.append([message, ""]) + + if self.agent is None: + asyncio.run(self.initialize_system()) + + response = self.agent.create_turn( + messages=[{"role": "user", "content": message}], + session_id=self.session_id, + ) + + current_response = "" + context_shown = False + + for log in EventLogger().log(response): + log.print() + if hasattr(log, "content"): + # Format context blocks if present + if not context_shown and "Retrieved context from banks" in str(log): + context = self.format_context(str(log)) + current_response = context + current_response + context_shown = True + else: + current_response += log.content + + history[-1][1] = current_response + yield history.copy() + + def format_context(self, log_str: str) -> str: + """Format the context block with custom styling.""" + # Extract context and clean up the markers + context_match = re.search( + r"Retrieved context from banks:.*?\n(.*?===.*?===.*?)(?=\n>|$)", + log_str, + re.DOTALL, + ) + if context_match: + context = context_match.group(1).strip() + # Remove the marker lines + context = re.sub( + r"====\s*Here are the retrieved documents for relevant context:\s*===\s*START-RETRIEVED-CONTEXT\s*===\s*", + "", + context, + flags=re.IGNORECASE, + ) + return f""" +
+
Retrieved Context
+
{context}
+
+""" + return "" + + +def create_gradio_interface( + host: str = HOST, + port: int = LLAMA_STACK_PORT, + docs_dir: str = DOCS_DIR, +): + chat_interface = LlamaChatInterface(host, port, docs_dir) + + with gr.Blocks(theme=gr.themes.Soft(), css=CUSTOM_CSS) as interface: + gr.Markdown("# LlamaStack Chat") + + chatbot = gr.Chatbot( + bubble_full_width=False, + show_label=False, + height=400, + container=True, + render_markdown=True, + ) + msg = gr.Textbox( + label="Message", + placeholder="Type your message here...", + show_label=False, + container=False, + ) + with gr.Row(): + submit = gr.Button("Send", variant="primary") + clear = gr.Button("Clear") + + gr.Examples( + examples=[ + "What topics are covered in the documents?", + "Can you summarize the main points?", + "Tell me more about specific details in the text.", + ], + inputs=msg, + ) + + def clear_chat(): + return [], "" + + submit_event = msg.submit( + fn=chat_interface.chat_stream, + inputs=[msg, chatbot], + outputs=chatbot, + queue=True, + ).then( + fn=lambda: "", + outputs=msg, + ) + + submit_click = submit.click( + fn=chat_interface.chat_stream, + inputs=[msg, chatbot], + outputs=chatbot, + queue=True, + ).then( + fn=lambda: "", + outputs=msg, + ) + + clear.click(clear_chat, outputs=[chatbot, msg], queue=False) + + msg.submit(lambda: None, None, None, api_name=False) + interface.load(fn=chat_interface.initialize_system) + + return interface + + +if __name__ == "__main__": + # Create and launch the Gradio interface + interface = create_gradio_interface() + interface.launch( + server_name=HOST, server_port=GRADIO_SERVER_PORT, share=True, debug=True + ) diff --git a/examples/DocQA/data/assets/DocQA.png b/examples/DocQA/data/assets/DocQA.png new file mode 100644 index 000000000..4d922b9e4 Binary files /dev/null and b/examples/DocQA/data/assets/DocQA.png differ diff --git a/examples/DocQA/data/eval/.env.template b/examples/DocQA/data/eval/.env.template new file mode 100644 index 000000000..304e30e2d --- /dev/null +++ b/examples/DocQA/data/eval/.env.template @@ -0,0 +1,14 @@ +# Server Configuration +HOST=localhost +PORT=5000 +CHROMA_PORT=8000 + +# Model and Memory Configuration +MODEL_NAME=meta-llama/Llama-3.2-3B-Instruct +MEMORY_BANK_ID=eval_bank + +# File Paths +DOCS_DIR=../output + +# Optional: Add your API keys here if needed +# OPENAI_API_KEY=your_api_key_here diff --git a/examples/DocQA/data/eval/QA_eval.json b/examples/DocQA/data/eval/QA_eval.json new file mode 100644 index 000000000..664bff1e2 --- /dev/null +++ b/examples/DocQA/data/eval/QA_eval.json @@ -0,0 +1,102 @@ +[ + { + "Question": "What is the policy regarding smoking in City offices?", + "Answer": "Smoking is not permitted in City offices, or within 20 feet of entrances, exits, or operable windows of public buildings. (Source: Page 46, 'Smoke-Free Workplace' section)" + }, + { + "Question": "How many days of paid sick leave do most full-time employees earn per year under Civil Service Rules?", + "Answer": "Most full-time employees earn 13 8-hour working days per year of paid sick leave under the Civil Service Rules. (Source: Page 32, 'Accrual of Paid Sick Leave' section)" + }, + { + "Question": "What are the three categories of employees eligible for health coverage?", + "Answer": "The following employees are eligible:\n\nAll permanent employees working at least 20 hours per week\n\nAll regularly scheduled provisional employees working at least 20 hours per week\n\nAll other employees (including temporary exempt or 'as needed') who have worked more than 1040 hours in any consecutive 12-month period and work at least 20 hours per week (Source: Page 25, 'Eligibility' section)" + }, + { + "Question": "How long must an employee wait before using vacation time after starting employment?", + "Answer": "Employees are not eligible to use vacation in the first year of continuous service. After one year of continuous service, they are awarded vacation allowance at the rate of .0385 of an hour for each hour of paid service in the preceding year. (Source: Page 30, 'Vacation' section)" + }, + { + "Question": "What must an employee do if they're summoned for jury duty?", + "Answer": "An employee must notify their supervisor as soon as they receive a jury summons. If required to report during working hours, they will be excused from work on the day they perform jury service, provided they give prior notification. If not selected or dismissed early, they must return to work as soon as possible. (Source: Page 37, 'Jury Duty Leave' section)" + }, + { + "Question": "What happens if an employee is absent without authorization for more than five consecutive working days?", + "Answer": "If an employee is absent from their job without proper authorization for more than five consecutive working days, or fails to return from an approved leave, their absence will be deemed an 'automatic resignation.' (Source: Page 19, 'Automatic Resignation' section)" + }, + { + "Question": "How long is the normal probationary period for permanent civil service positions?", + "Answer": "The document states that all appointments to permanent civil service positions require a probationary period, but the duration is governed by the collective bargaining agreement. Absences from work will extend the probationary period. (Source: Page 14, 'Probationary Period' section)" + }, + { + "Question": "What are employees required to do in case of a catastrophic event while off duty?", + "Answer": "Employees should ensure the safety of their family and follow their department's instructions. If phone lines are down, they are required to listen to the radio for any reporting instructions. (Source: Page 51, 'Catastrophic Event While off Duty' section)" + }, + { + "Question": "What is the city's policy on accepting gifts from subordinates?", + "Answer": "Employees may not solicit or accept any gifts from any subordinate, or any candidate or applicant for a position as an employee or subordinate to them. (Source: Page 49, 'Gifts' section)" + }, + { + "Question": "What documents must new employees present at the time of hire?", + "Answer": "New employees must present:\n\nSocial Security number\n\nDocuments to verify identity and authorization to work in the United States\n\nTake the Oath of Office\n\nSign acknowledgment of no-strike provision\n\nComplete tax forms (W-4)\n\nComplete Warrant Recipient Form (Source: Pages 11-12, 'Requirements at the Time of Hire' section)" + }, + { + "Question": "What are the fees for a new Special Traffic Permit in San Francisco?", + "Answer": "New Special Traffic Permits cost $388 processing fee plus $78 per day." + }, + { + "Question": "During what hours does the Holiday Moratorium restrict construction work in San Francisco?", + "Answer": "No construction work is allowed in the public right of way from the day after Thanksgiving to January 1, inclusive, during the hours of 7 a.m. to 10 p.m. on Holiday Moratorium protected streets." + }, + { + "Question": "How many days in advance must contractors register tow-away signs with SFMTA?", + "Answer": "Tow-Away signs must be posted and registered with the SFMTA Tow-Desk or SF Public Works online at least 72 hours prior to the enforcement date." +}, +{ + "Question": "What is the minimum width required for a single traffic lane in San Francisco?", + "Answer": "The minimum width required for a single traffic lane is 12 feet. The minimum width required for a vehicular traffic lane contiguous to one or more vehicular traffic lanes in the same direction is 10 feet." +}, +{ + "Question": "What clearance must equipment maintain from Muni overhead electric wires?", + "Answer": "CAL OSHA and CPUC General Order 95 regulations require that any boom type equipment that moves vertically must maintain a 10 feet radial clearance and any other equipment must maintain a six (6) foot clearance from Muni overhead wires." +}, +{ + "Question": "What restrictions apply around Oracle Park during events?", + "Answer": "No construction activity is allowed from two (2) hours before to two (2) hours after these events around Oracle Park." +}, +{ + "Question": "What is the minimum processing time required for new Special Traffic Permits?", + "Answer": "New permits may require five (5) working days. More complicated requests may take longer. Very complex permit applications, such as Tower Crane Erections, Concrete Mat Pours and Complex road closures require a minimum of 8 weeks processing time." +}, +{ + "Question": "What is the minimum clear path of travel required on sidewalks during construction?", + "Answer": "A 6-foot wide clear path of travel is desirable. At a minimum, Contractor shall provide a 4-foot wide clear path of travel on any sidewalk at all times." +}, +{ + "Question": "What are the main functions of the Legislative Department regarding Bills?", + "Answer": "According to Section 1.1(ii) of the document, the Legislative Department's main functions regarding Bills include: 'Drafting and scrutiny of all Government Bills including Constitution (Amendment) Bills, translation of all the Bills into Hindi and forwarding of both English and Hindi versions of the Bills to the Lok Sabha or Rajya Sabha Secretariat for introduction in Parliament; drafting of official amendments to the Bills; scrutiny of non-official amendments and rendering assistance to administrative Ministries/Departments to decide the acceptability or otherwise of non-official amendments.'" +}, +{ + "Question": "How many Bills were forwarded to Parliament for introduction during the period from January 1, 2021 to December 31, 2021?", + "Answer": "According to Section 4 of the document, 50 Legislative Bills were forwarded to Parliament for introduction during this period. The exact quote states: '50 Legislative Bills were forwarded to Parliament for introduction during this period.'" +}, +{ + "Question": "What is the role of the Official Languages Wing regarding legal terminology?", + "Answer": "According to Section 41(1)(i) of the document, the Official Languages Wing is responsible for 'Preparation and publication of a standard legal terminology for use, as far as possible, in all Official Languages.' The Wing has published seven editions of Legal Glossary, with the latest 7th Edition containing approximately 65,000 entries." +}, +{ + "Question": "How does the Legislative Department handle public grievances?", + "Answer": "According to Section 45 of the document, 'During the period from 1st January, 2021 to 31st December, 2021 Legislative Department received 1644 public grievances on CPGRAMS portal. Further 223 public grievances were pending before 1st January, 2021. During the said period 1544 grievances have been disposed off and action is being taken for disposal of remaining grievances on priority basis.'" +}, +{ + "Question": "What are the provisions for linking electoral rolls with Aadhaar according to the Election Laws (Amendment) Act, 2021?", + "Answer": "According to Section 22.1 of the document, 'The newly passed Election Laws (Amendment) Act, 2021 amends both The Representation of the People Acts of 1950 and 1951. One of its main provisions creates a legal framework to link electoral roll data with Aadhaar, the unique identification number. It is meant to strike out bogus voters, dead voters, foreigners who are wrongfully included as voters and also those who are enrolled in more than one constituency.'" +}, +{ + "Question": "What role does the Printing Section play in the Legislative Department?", + "Answer": "According to Section 36 of the document, the Printing Sections (Printing I and II) handle 'the processing of legislation for printing at various stages' including 'editing of manuscripts of the Bills (including preparation of contents and annexures), Ordinances, Regulations, Adaptation Orders, Orders issued under the Constitution of India, Delimitation Orders and other statutory instruments before sending them to Press.' During January-December 2021, they edited manuscripts and checked proofs for 89 Bills, 49 Gazettes, 10 Ordinances, and prepared 49 A-4 Acts." +}, + { + "Question": "How many Official Language versions of the Constitution of India have been published?", + "Answer": "According to Section 41(3) of the document, the Constitution of India has been published in Hindi (the Official Language of the Union) and 16 other regional languages: 'Assamese, Bengali, Gujarati, Kannada, Malayalam, Marathi, Manipuri, Oriya, Punjabi, Sanskrit, Tamil, Telugu, Urdu, Sindhi, Nepali and Konkani.'" + } +] diff --git a/examples/DocQA/data/eval/README.md b/examples/DocQA/data/eval/README.md new file mode 100644 index 000000000..ba042b9cd --- /dev/null +++ b/examples/DocQA/data/eval/README.md @@ -0,0 +1,98 @@ +# RAG System Evaluation + +This directory contains tools for evaluating the Retrieval-Augmented Generation (RAG) system using RAGAS metrics. + +## Setup + +1. Create your environment file: + +```bash +cp .env.template .env +``` + +2. Configure the environment variables in `.env`: + +```env +# Server Configuration +HOST=localhost # Your server host +PORT=5000 # Your server port +CHROMA_PORT=8000 # Chroma DB port + +# Model and Memory Configuration +MODEL_NAME=meta-llama/Llama-3.2-3B-Instruct # Model to use +MEMORY_BANK_ID=eval_bank # Memory bank identifier + +# File Paths +DOCS_DIR=../output # Directory containing your documents +``` + +## Running the Evaluation + +1. Make sure your server is running and accessible at the configured host and port. + +2. Run the evaluation script: + +```bash +python eval.py +``` + +The script will: + +- Set up a memory bank for evaluation +- Load your documents +- Generate responses for test questions +- Evaluate the responses using various RAGAS metrics: + - Context Precision + - Context Recall + - Faithfulness + - Answer Relevancy + - Factual Correctness + - Semantic Similarity + +Results will be saved to `evaluation_results.csv`. + +## Analyzing Results + +For detailed analysis of your evaluation results, you can use the Jupyter notebook: + +```bash +jupyter notebook explain-eval.ipynb +``` + +The notebook provides: + +- Visualization of evaluation metrics +- Detailed breakdown of each metric +- Analysis of system performance +- Insights for improvement + +## Metrics Explanation + +The evaluation uses the following RAGAS metrics: + +1. **Context Precision**: Measures how much of the retrieved context is actually relevant +2. **Context Recall**: Measures if all relevant information was retrieved +3. **Faithfulness**: Measures if the answer is faithful to the provided context +4. **Answer Relevancy**: Measures if the answer is relevant to the question +5. **Factual Correctness**: Measures the factual accuracy of the answer +6. **Semantic Similarity**: Measures semantic closeness between answer and reference + +## Troubleshooting + +If you encounter issues: + +1. Verify your server is running and accessible +2. Check the environment variables in `.env` +3. Ensure your documents are in the correct directory +4. Check the logs for detailed error messages + +## Requirements + +- Python 3.10+ +- Jupyter Notebook (for analysis) +- Required Python packages (install via `pip`): + - ragas + - datasets + - pandas + - numpy + - matplotlib (for visualization) diff --git a/examples/DocQA/data/eval/eval.py b/examples/DocQA/data/eval/eval.py new file mode 100644 index 000000000..4a193ca07 --- /dev/null +++ b/examples/DocQA/data/eval/eval.py @@ -0,0 +1,321 @@ +import asyncio +import json +import logging +import os +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Tuple, Dict, Any + +import chromadb +import fire +import requests +from datasets import Dataset +from dotenv import load_dotenv +from llama_stack_client import LlamaStackClient +from llama_stack_client.lib.agents.agent import Agent +from llama_stack_client.types.agent_create_params import AgentConfig +from llama_stack_client.types.memory_insert_params import Document +from ragas import evaluate +from ragas.metrics import ( + AnswerRelevancy, + ContextPrecision, + ContextRecall, + FactualCorrectness, + Faithfulness, + SemanticSimilarity, +) +from tqdm import tqdm + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + + +@dataclass +class Config: + """Configuration for the evaluation script.""" + host: str + port: int + chroma_port: int + model_name: str + memory_bank_id: str + docs_dir: Path + + @classmethod + def from_env(cls) -> 'Config': + """Create configuration from environment variables.""" + load_dotenv() + return cls( + host=os.getenv("HOST", "localhost"), + port=int(os.getenv("PORT", 5000)), + chroma_port=int(os.getenv("CHROMA_PORT", 8000)), + model_name=os.getenv( + "MODEL_NAME", "meta-llama/Llama-3.2-3B-Instruct"), + memory_bank_id=os.getenv("MEMORY_BANK_ID", "test_bank_236"), + docs_dir=Path(os.getenv("DOCS_DIR", "../output")).resolve(), + ) + + +class MemoryBankManager: + """Manages memory bank operations.""" + + def __init__(self, client: LlamaStackClient, config: Config): + self.client = client + self.config = config + + async def setup(self) -> None: + """Set up the memory bank if it doesn't exist.""" + try: + providers = self.client.providers.list() + provider_id = providers["memory"][0].provider_id + memory_banks = self.client.memory_banks.list() + + if any(bank.identifier == self.config.memory_bank_id for bank in memory_banks): + logger.info( + f"Memory bank '{self.config.memory_bank_id}' exists.") + return + + logger.info( + f"Creating memory bank '{self.config.memory_bank_id}'...") + self.client.memory_banks.register( + memory_bank_id=self.config.memory_bank_id, + provider_id=provider_id, + ) + await self._load_documents() + logger.info("Memory bank registered successfully.") + + except Exception as e: + logger.error(f"Failed to setup memory bank: {str(e)}") + raise + + async def _load_documents(self) -> None: + """Load documents from the specified directory into memory bank.""" + try: + documents = [] + for file_path in self.config.docs_dir.glob("*.{txt,md}"): + document = self._create_document(file_path) + if document: + documents.append(document) + + if documents: + self.client.memory.insert( + bank_id=self.config.memory_bank_id, + documents=documents, + ) + logger.info( + f"Loaded {len(documents)} documents from {self.config.docs_dir}") + + except Exception as e: + logger.error(f"Failed to load documents: {str(e)}") + raise + + def _create_document(self, file_path: Path) -> Optional[Document]: + """Create a Document object from a file.""" + try: + content = file_path.read_text(encoding="utf-8") + return Document( + document_id=file_path.name, + content=content, + mime_type="text/plain", + metadata={"filename": file_path.name}, + ) + except Exception as e: + logger.error( + f"Failed to create document from {file_path}: {str(e)}") + return None + + def query(self, query: str) -> Optional[Dict[str, Any]]: + """Query memory bank for relevant context.""" + try: + response = self.client.memory.query( + bank_id=self.config.memory_bank_id, + query=[query], + ) + + if response.chunks and response.scores: + return { + "documents": [chunk.content for chunk in response.chunks], + "metadatas": [{"content": chunk.content} for chunk in response.chunks], + "distances": response.scores + } + return None + + except Exception as e: + logger.error(f"Failed to query memory: {str(e)}") + return None + + +class ResponseGenerator: + """Handles generation of responses using the agent.""" + + def __init__(self, agent: Agent, memory_manager: MemoryBankManager): + self.agent = agent + self.memory_manager = memory_manager + + async def get_response(self, query: str, session_id: str) -> Tuple[str, List[str]]: + """Generate a response for the given query using context from memory.""" + try: + context, contexts = self._get_context(query) + messages = [ + {"role": "user", "content": f"Context: {context}\n\nQuestion: {query}"}] + + response = self.agent.create_turn( + messages=messages, session_id=session_id) + return self._process_response(response), contexts + + except Exception as e: + logger.error(f"Failed to generate response: {str(e)}") + return "Error generating response.", [] + + def _get_context(self, query: str) -> Tuple[str, List[str]]: + """Get context for the query from memory.""" + results = self.memory_manager.query(query) + if results and results["metadatas"]: + contexts = [metadata["content"] + for metadata in results["metadatas"]] + context = "\n".join(f"Content:\n{ctx}" for ctx in contexts) + return context, contexts + return "No relevant context found.", [] + + def _process_response(self, response) -> str: + """Process the response from the agent.""" + full_response = "" + for chunk in response: + if hasattr(chunk, "event"): + if chunk.event.payload.event_type == "turn_complete": + return chunk.event.payload.turn.output_message.content + elif hasattr(chunk.event.payload, "delta"): + full_response += chunk.event.payload.delta.content or "" + return full_response or "No response generated." + + +class Evaluator: + """Handles evaluation of the question-answering system.""" + + def __init__(self, config: Config): + self.config = config + + async def run_evaluation(self) -> None: + """Run the evaluation process.""" + try: + client = self._setup_client() + memory_manager = MemoryBankManager(client, self.config) + await memory_manager.setup() + + agent = self._setup_agent(client) + response_generator = ResponseGenerator(agent, memory_manager) + + qa_data = self._load_qa_data() + results = await self._process_questions(response_generator, qa_data) + self._evaluate_and_save_results(results) + + except Exception as e: + logger.error(f"Evaluation failed: {str(e)}") + raise + + def _setup_client(self) -> LlamaStackClient: + """Set up the LlamaStack client.""" + return LlamaStackClient(base_url=f"http://{self.config.host}:{self.config.port}") + + def _setup_agent(self, client: LlamaStackClient) -> Agent: + """Set up the agent with configuration.""" + agent_config = AgentConfig( + model=self.config.model_name, + instructions="You are a helpful assistant that can answer questions based on provided documents.", + sampling_params={"strategy": "greedy", + "temperature": 1.0, "top_p": 0.9}, + tools=[{ + "type": "memory", + "memory_bank_configs": [{"bank_id": self.config.memory_bank_id, "type": "vector"}], + }], + tool_choice="auto", + tool_prompt_format="json", + enable_session_persistence=True, + ) + return Agent(client, agent_config) + + def _load_qa_data(self) -> List[Dict[str, str]]: + """Load QA evaluation data.""" + qa_file_path = Path(__file__).parent / "QA_eval.json" + with qa_file_path.open('r') as f: + return json.load(f)[:10] + + async def _process_questions( + self, + response_generator: ResponseGenerator, + qa_data: List[Dict[str, str]] + ) -> Dict[str, List]: + """Process all questions and collect results.""" + results = { + "questions": [], + "generated_answers": [], + "retrieved_contexts": [], + "ground_truths": [] + } + + session_id = f"session-{uuid.uuid4()}" + for qa in tqdm(qa_data, desc="Generating responses"): + try: + question = qa["Question"] + ground_truth = qa["Answer"] + + answer, contexts = await response_generator.get_response(question, session_id) + + results["questions"].append(question) + results["generated_answers"].append(answer) + results["retrieved_contexts"].append( + [str(ctx) for ctx in contexts]) + results["ground_truths"].append(ground_truth) + + except Exception as e: + logger.error(f"Failed to process question: {str(e)}") + continue + + return results + + def _evaluate_and_save_results(self, results: Dict[str, List]) -> None: + """Evaluate and save the results.""" + eval_data = Dataset.from_dict({ + "user_input": results["questions"], + "response": results["generated_answers"], + "retrieved_contexts": results["retrieved_contexts"], + "reference": results["ground_truths"], + }) + + metrics = [ + ContextPrecision(), + ContextRecall(), + Faithfulness(), + AnswerRelevancy(), + FactualCorrectness(), + SemanticSimilarity(), + ] + + evaluation_result = evaluate(eval_data, metrics=metrics) + df = evaluation_result.to_pandas() + df.to_csv("evaluation_results.csv", index=False) + logger.info("\nEvaluation Results:") + logger.info("\n" + str(df)) + + +async def run_main(docs_dir: str = None) -> None: + """Main entry point for the evaluation script.""" + config = Config.from_env() + if docs_dir: + config.docs_dir = Path(docs_dir).resolve() + + evaluator = Evaluator(config) + await evaluator.run_evaluation() + + +def main() -> None: + """CLI entry point.""" + asyncio.run(run_main()) + + +if __name__ == "__main__": + fire.Fire(main) diff --git a/examples/DocQA/data/eval/evaluation_results.csv b/examples/DocQA/data/eval/evaluation_results.csv new file mode 100644 index 000000000..09b235f37 --- /dev/null +++ b/examples/DocQA/data/eval/evaluation_results.csv @@ -0,0 +1,29 @@ +user_input,retrieved_contexts,response,reference,context_precision,context_recall,faithfulness,answer_relevancy,factual_correctness,semantic_similarity +What is the policy regarding smoking in City offices?,"[' productive workplace which is free from inappropriate workplace behavior.\n\n## Smoke-Free Workplace\n\nSmoking is not permitted in City offices, or within n 20 feet of entrances, exits, or operable windows of public buildings.\n\n## Drug-Free Workplace\n\nYou may not manufacture, distribute, dispense, possess, use or be under the influence of alcohol or illegal drugs in workplace. This prohibition includes prescription drugs used improperly (e.g., those not prescribed for the user). Any violation of this policy may be grounds', '45 |\n| Policy Regarding the Treatment of Co-Workers and Members of the Public ...................46 |\n| Smoke-Free Workplace .....................................................................................................46 |\n| Drug-Free Workplace ........................................................................................................46 Disciplinary Action against Striking Employees ...............................................................47 |\n| Political Activity ................................................................................................................47 |\n\n| If You Suspect Improper or Criminal Activity on the Job .................................................47 |\n|---------------------------------------------------------------------------------------------------------------------------------------------|\n| Use of City and County Property for Business Purposes', ' City resources or divert employees from their assigned duties.\n\nCity employees are prohibited from using their official positions to influence elections, and from using City funds or resources for political or r election activities. Further, City employees may not participate in political activities of any kind while in uniform (i.e., part or all of a uniform they are required or authorized to wear when engaged in official duties).\n\nViolation of these rules may result in considerable civil and criminal penalties, as well as discipline, up to and including']",No response generated.,"Smoking is not permitted in City offices, or within 20 feet of entrances, exits, or operable windows of public buildings. (Source: Page 46, 'Smoke-Free Workplace' section)",0.99999999995,0.5,0.0,0.0,0.0,0.7310965383185614 +How many days of paid sick leave do most full-time employees earn per year under Civil Service Rules?,"[' Accrual of Paid Sick Leave\n\nSick leave with pay is a privilege under the Civil Service Rules, Charter and City Ordinance.\n\nMost full time employees earn 13 8-hour working days per year of paid sick leave under the Civil Service Rules. Unused sick leave accrued under the Civil Service Rules may be accumulated from year to year up to a maximum of 130 8-hour working days.\n\nSome employees may be entitled to accrue sick leave under Chapter 12W of the Administrative', ' must complete the appropriate form and comply with the specific requirements for that type of leave. All requests for leave of more than five working days must be made in writing on the appropriate form. Please see your departmental personnel officer for more information .\n\n## Sick Leave\n\nPlease refer to your collective bargaining agreement and the Civil Service Rules or contact your departmental personnel officer for more information regarding sick leave accrual, usage, and notice and verification requirements.\n\n## Accrual of Paid Sick Leave\n\nSick', ' leave under Chapter 12W of the Administrative Code. Please contact your departmental personnel officer for more information.\n\n## Sick Leave without Pay\n\nYou may be eligible to take sick leave without pay. Please refer to the Civil Service Rules and any applicable collective bargaining agreement for more information.\n\n## Use of Sick Leave\n\nYou may not use sick leave with pay accrued under the Civil Service Rules during your first 6 months of employment with the City.\n\nYou may not use sick leave for reasons not set']",No response generated.,"Most full-time employees earn 13 8-hour working days per year of paid sick leave under the Civil Service Rules. (Source: Page 32, 'Accrual of Paid Sick Leave' section)",0.9999999999,0.5,0.0,0.0,0.0,0.7089057204017655 +What are the three categories of employees eligible for health coverage?,"[' health coverage:\n\n- (1) All permanent employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours;\n- (2) All regularly scheduled provisional employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours; and\n- (3) All other employees of the City including temporary y exempt or ""as needed"" employees who have worked more than 1040 hours in any consecutive ', '02.\n\n## EMPLOYEE BENEFITS\n\nThe City provides eligible employees and their eligible dependents the opportunity to enroll in medical, dental, vision and flexible spending account benefits, administered by the Health Service System (""HSS"").\n\nPlease note that the information in this section is subject to change. You should consult with HSS if you have any questions at (415) 554-1750.\n\n## Eligibility\n\nThe following employees are eligible for health coverage:\n\n- (1) All permanent employees', ' of the employee organization, is available at the discretion of the department head.\n\nWhile the employee is on leave, the employee organization is responsible for paying the employee\'s salary.\n\nBefore the leave begins, the employee may choose to either continue or waive access to health coverage through the City for the duration of the approved leave. If the employee wishes to continue his or her health coverage, premium contributions for the employee and any covered dependents must be paid directly to the Health Services System (""HSS"").']",No response generated.,"The following employees are eligible: + +All permanent employees working at least 20 hours per week + +All regularly scheduled provisional employees working at least 20 hours per week + +All other employees (including temporary exempt or 'as needed') who have worked more than 1040 hours in any consecutive 12-month period and work at least 20 hours per week (Source: Page 25, 'Eligibility' section)",0.99999999995,1.0,0.0,0.0,0.0,0.6871705187744059 +How long must an employee wait before using vacation time after starting employment?,"[', you will be awarded a vacation allowance at the rate of .0385 of an hour for each hour of paid service in the preceding year and will be eligible to use accrued vacation time. For the purpose of computing vacation, most employees may be credited with no more than 2080 hours of service in a 12month period.\n\nAn additional vacation entitlement will be awarded after five years of continuous service, usually 40 hours for full-time employees. After fifteen years of service, full-time employees will', 'ing officer and the provisions of the applicable collective bargaining agreement. Contact your departmental personnel officer or payroll representative for more information.\n\n## Vacation\n\nThe City provides annual vacations for employees who work a regular schedule and who have completed one year or more of continuous service. Certain temporary exempt employees may also be eligible for vacation benefits. You are not eligible to use vacation in the first year of continuous service; however, at the end of one year of continuous service, you will be awarded a vacation allowance at the', "" department's procedures for details.\n\nWhen a holiday falls on a regular working day during your vacation, the holiday is not counted as a vacation day. If you resign or are separated after one or more years of continuous service and have not taken all of your vacation, you will be paid for the accumulated unused vacation.\n\nRequests to take vacation must be made and approved d in advance, in accordance with your department's procedures.\n\n## LEAVES OF ABSENCE\n\nLeaves of absence are governed by the""]",No response generated.,"Employees are not eligible to use vacation in the first year of continuous service. After one year of continuous service, they are awarded vacation allowance at the rate of .0385 of an hour for each hour of paid service in the preceding year. (Source: Page 30, 'Vacation' section)",0.99999999995,0.6666666666666666,0.0,0.0,0.0,0.710133331048373 +What must an employee do if they're summoned for jury duty?,"[' you would return to the workplace to work the remaining two hours each day and continue to take Friday off.\n\nYou are not entitled to a per diem pay from the City or County for which Jury Duty was served, as the City and County of San Francisco already provides regular compensation and benefits to you while performing jury service. You must notify the jury staff that you are a government employee when reporting for jury service\n\n## Witness Duty\n\nIf you are summoned as a witness on behalf of the City and', ' 10 days of unpaid leave during a period of his or her leave from deployment. Please refer to the Civil Service Rules for details or consult your departmental personnel officer for more information.\n\n## Jury Duty Leave\n\nYou must notify your supervisor as soon as you receive a jury summons. If you are required to report for jury duty during your working hours, you will be excused from work on the work day you perform jury service, provided you give prior notification to your supervisor. If you report', ' give prior notification to your supervisor. If you report for jury duty and are not selected as a juror, or if the court dismisses the proceedings early for the day, you must return to work as soon as possible.\n\n## Work Schedule While on Jury Duty\n\nAlternative Work Schedule. If you have an alternative work schedule and perform jury service for an extended period, your work schedule will revert to a regular Monday through Friday work schedule for compensation purposes for the duration of the jury service.\n\nSwing and']",No response generated.,"An employee must notify their supervisor as soon as they receive a jury summons. If required to report during working hours, they will be excused from work on the day they perform jury service, provided they give prior notification. If not selected or dismissed early, they must return to work as soon as possible. (Source: Page 37, 'Jury Duty Leave' section)",0.5833333333041666,0.75,0.0,0.0,0.0,0.7250633971841919 +What happens if an employee is absent without authorization for more than five consecutive working days?,"[' work and the number of hours worked in every work day.\n\nAll planned absences must be requested and approved d in advance. If illness or some other emergency causes an unplanned or unforeseeable absence, you must notify your department as soon as possible on the first day of absence, and keep the department advised daily during the absence. In the case of an extended unforeseeable absence, you may be asked to complete forms and submit medical certifications as appropriate during your leave. Improper use of sick', 'ation\n\nIf you are absent from your job for any period of time without proper authorization, you may be subject to discipline. If you are absent from your r job without proper authorization for more than five consecutive working days, or if you fail to return from an approved leave, your absence will be deemed an ""automatic resignation."" If you receive a notice of automatic resignation and wish to appeal the determination to the Civil Service Commission, you will have fifteen days from the date on which the notice was mailed to', ' period of incarceration .\n\nPlease see your departmental personnel officer if you have any questions regarding appropriate uses of sick leave.\n\n## Notice and Verification Requirements\n\nIt is your responsibility to notify your supervisor r as soon as possible whenever you are unable to report for work due to illness. You must t keep your supervisor informed throughout your absence and notify your supervisor of your expected date of return to work.\n\nIf you are absent from your job for more than five consecutive working days, you must submit to your supervisor a completed']",No response generated.,"If an employee is absent from their job without proper authorization for more than five consecutive working days, or fails to return from an approved leave, their absence will be deemed an 'automatic resignation.' (Source: Page 19, 'Automatic Resignation' section)",0.49999999995,1.0,0.0,0.0,0.0,0.7395137950936732 +How long is the normal probationary period for permanent civil service positions?,"["" and most important phase of the selection process. This period is used to evaluate your performance and suitability for the position. The duration of the probationary period is governed by the collective bargaining agreement. Absences from work will extend your probationary period. For infoformation about the length of your probationary period, consult your departmental personnel officer or collective bargaining agreement.\n\nThe department's appointing officer may release an employee at any time during the probationary period. At the conclusion of the probationary period,"", ' of eligibles to a permanent position. Permanent empmployees who have successfully completed the probationary period may only be removed for cause.\n\n## Exempt\n\nPersons appointed to temporary or permanent positions exempted from civil service status in accordance with the City Charter. Exempt t appointees serve at the pleasure of the appointing officer.\n\n## Provisional\n\nProvisional appointments are made when an eligible list for the classification is not available. The just cause rights of provisional employees are described in the', ' the class from m which they resigned within four years from the date of resignation. Former members of the uniformed ranks of the Police and Fire departments may be reappointed within two years of the date of resignation. Former employees may request reappointment to a vacancy in any class in which the employee has completed the probationary period. The receiving department, at i its discretion, may approve a request for reappointment.\n\n## Reinstatement\n\nAn employee may submit a written request to return to a vacant']",No response generated.,"The document states that all appointments to permanent civil service positions require a probationary period, but the duration is governed by the collective bargaining agreement. Absences from work will extend the probationary period. (Source: Page 14, 'Probationary Period' section)",0.99999999995,0.6666666666666666,0.0,0.0,0.0,0.7124925328695783 +What are employees required to do in case of a catastrophic event while off duty?,"[' whatever role you are directed to perform by a superior or by law.\n\nAll Disaster Services Workers will be issued a citywide identification badge. You must have your badge with you at all times.\n\nCatastrophic Event While on Duty: Should a catastrophic event occur while you are on duty, report immediately to your supervisor or designated areas for assignment.\n\nCatastrophic Event While off Duty: Should a catastrophic event occur while you are off duty, you should ensure the safety of your family and follow your', ' less.\n\n## Disaster Service Workers - Your Responsibilities in Case of Emergency\n\nAll City employees are designated Disaster Service Workers under state and local law. If the Mayor or an authorized City official proclaims a local emergency due to actual or threatened disaster such as earthquake, fire, riot, etc., City employees, as Disaster Service Workers, may be required to provide services during the emergency and subsequent period of assessment and recovery.\n\nSimply put, being a Disaster Service Worker means that any time a catastrophic event-', "" healthful work environment.\n\nSafety is every City employee's responsibility. All employees are required to remain alert and to correct hazardous conditions and unsafe acts-if it is safe to do so-and to report unsafe conditions to their supervisors.\n\n## On-the-Job Injury and Workers' Compensation\n\nAll City employees are covered by State Workers' Compensation laws. If you are injured at work or because of your work, you must report the injury or illness to your immediate supervisor as soon as possible, and no later than the""]",No response generated.,"Employees should ensure the safety of their family and follow their department's instructions. If phone lines are down, they are required to listen to the radio for any reporting instructions. (Source: Page 51, 'Catastrophic Event While off Duty' section)",0.99999999995,0.5,0.0,0.0,0.0,0.7465800852696091 +What is the city's policy on accepting gifts from subordinates?,"[' a copy of your department\'s ""Statement of Incompatible Activities,"" please see your departmental personnel officer or go to the Ethics Commission website at www.sfgov.org/site/ethics.\n\n## Gifts\n\nYou may not accept any gift that is intended to inflfluence you in the performance of your job.\n\nYou may not solicit or accept any gift from any person or entity who has a contract with your department or who has attempted to influence you in a governmental decision during the past 12 months', ' a governmental decision during the past 12 months.\n\nYou may not solicit or accept any gifts from any subordinate, or any candidate or applicant for a position as an employee or subordinate to you .\n\nIf you are required to file a Statement of Economic Interests (""SEI"") financial disclosure, you cannot accept gifts worth more than a certain amount in a calendar year from any source that you must report on your SEI. That amount is $420.00 for calendar year 2011; please', ' recipient to believe that you are speaking in an official capacity when you are not.\n- You may not make, participate in, or attempt to inflfluence a governmental decision affecting a person or entity with whom you are discussing or negotiating an agreement concerning future employment.\n- You may not accept any compensation, reward, or gift ft from any source except the City for any service, advice, assistance or other matter related to your City job.\n- You may not solicit or accept anything of value in']",No response generated.,"Employees may not solicit or accept any gifts from any subordinate, or any candidate or applicant for a position as an employee or subordinate to them. (Source: Page 49, 'Gifts' section)",0.9999999999666667,0.5,0.0,0.0,0.0,0.7311572952124511 +What documents must new employees present at the time of hire?,"['\n\nPrior to employment in certain positions or classififications, departments may require a background review of criminal history, motor vehicle (driving) record, personnel history and employment records. The requirement for a background review will be published on the examination announcement.\n\n## Medical Examination\n\nDepending on the classification or position in which you are to be employed, you may be required to pass a medical examination, including drug testing, as a condition of employment. Most medical examinations are conducted by the Deparartment of Public', ' these documents may result in loss of eligibility. Acceptable verification documents are listed in the information pamphlet entitled ""Federal Immigration and Naturalization Service Requirements,"" available at the Department of Human Resources\' (""DHR"") Employment Information Center, located on the 4 th floor at 1 South Van Ness Avenue.\n\nAny applicant or employee seeking to have the City p provide documentation to the Department of Homeland Security (including the U.S. Citizenship and Immigration Services) about possible or actual employment with the City must obtain', 'ingerprinting\n\nAll employees must be fingerprinted. Fingerprints are electronically transmitted to the California Department of Justice to obtain conviction records.\n\n## REQUIREMENTS AT THE TIME OF HIRE\n\n## Social Security Number\n\nYou must have a Social Security number to work for the City.\n\n## Authorization to Work\n\nYou must present documents to verify identity and authorization to work in the United States as required by the Immigration Reform Control Act of 1986. Failure to provide these documents may result in loss of eligibility. Accept']",No response generated.,"New employees must present: + +Social Security number + +Documents to verify identity and authorization to work in the United States + +Take the Oath of Office + +Sign acknowledgment of no-strike provision + +Complete tax forms (W-4) + +Complete Warrant Recipient Form (Source: Pages 11-12, 'Requirements at the Time of Hire' section)",0.3333333333,0.3333333333333333,0.0,0.0,0.0,0.7086797649178207 diff --git a/examples/DocQA/data/eval/explain-eval.ipynb b/examples/DocQA/data/eval/explain-eval.ipynb new file mode 100644 index 000000000..8d90fac04 --- /dev/null +++ b/examples/DocQA/data/eval/explain-eval.ipynb @@ -0,0 +1,745 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "### **RAGAS Evaluation Results Analysis**\n", + "\n", + "Let's anaylze our RAG based on these four metrics, Context Precision, Context Recall, Faithfullness, and Answer Relevancy." + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "\n", + "sns.set_style('whitegrid')\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "#### Understanding the Metrics\n", + "\n", + "**Context Precision**\n", + "\n", + "- **Definition**: Measures the proportion of relevant chunks in the retrieved contexts.\n", + "- **Interpretation**: A higher value indicates that most of the retrieved contexts are relevant to the user query.\n", + "\n", + "**Context Recall**\n", + "\n", + "- **Definition**: Measures how many of the relevant documents were successfully retrieved.\n", + "- **Interpretation**: A higher value means fewer relevant documents were left out.\n", + "\n", + "**Faithfulness**\n", + "\n", + "- **Definition**: Measures the factual consistency of the generated answer against the given context.\n", + "- **Interpretation**: A higher value indicates that the answer is well-supported by the retrieved contexts.\n", + "\n", + "**Answer Relevancy**\n", + "\n", + "- **Definition**: Assesses how pertinent the generated answer is to the given prompt.\n", + "- **Interpretation**: Higher scores indicate that the answer directly addresses the user's question." + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "#### Data Preparation\n", + "\n", + "Lets visualize our dataset before we dive into the metrics." + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
user_inputretrieved_contextsresponsereferencecontext_precisioncontext_recallfaithfulnessanswer_relevancy
0What is the policy regarding smoking in City o...['## Smoke-Free Workplace\\n\\nSmoking is not pe...According to the document, smoking is not perm...Smoking is not permitted in City offices, or w...1.0000001.0000001.00.968476
1How many days of paid sick leave do most full-...['Most full time employees earn 13 8-hour work...Most full-time employees earn 13 8-hour workin...Most full-time employees earn 13 8-hour workin...1.0000000.5000001.00.986973
2What are the three categories of employees eli...['The following employees are eligible for hea...The three categories of employees eligible for...The following employees are eligible:\\n\\nAll p...0.7000001.0000000.80.981254
3How long must an employee wait before using va...['An additional vacation entitlement will be a...An employee must wait at least one year of con...Employees are not eligible to use vacation in ...0.4166670.6666671.00.979251
4What must an employee do if they're summoned f...['You must notify your supervisor as soon as y...An employee must notify their supervisor as so...An employee must notify their supervisor as so...0.9166670.7500001.00.965414
\n", + "
" + ], + "text/plain": [ + " user_input \\\n", + "0 What is the policy regarding smoking in City o... \n", + "1 How many days of paid sick leave do most full-... \n", + "2 What are the three categories of employees eli... \n", + "3 How long must an employee wait before using va... \n", + "4 What must an employee do if they're summoned f... \n", + "\n", + " retrieved_contexts \\\n", + "0 ['## Smoke-Free Workplace\\n\\nSmoking is not pe... \n", + "1 ['Most full time employees earn 13 8-hour work... \n", + "2 ['The following employees are eligible for hea... \n", + "3 ['An additional vacation entitlement will be a... \n", + "4 ['You must notify your supervisor as soon as y... \n", + "\n", + " response \\\n", + "0 According to the document, smoking is not perm... \n", + "1 Most full-time employees earn 13 8-hour workin... \n", + "2 The three categories of employees eligible for... \n", + "3 An employee must wait at least one year of con... \n", + "4 An employee must notify their supervisor as so... \n", + "\n", + " reference context_precision \\\n", + "0 Smoking is not permitted in City offices, or w... 1.000000 \n", + "1 Most full-time employees earn 13 8-hour workin... 1.000000 \n", + "2 The following employees are eligible:\\n\\nAll p... 0.700000 \n", + "3 Employees are not eligible to use vacation in ... 0.416667 \n", + "4 An employee must notify their supervisor as so... 0.916667 \n", + "\n", + " context_recall faithfulness answer_relevancy \n", + "0 1.000000 1.0 0.968476 \n", + "1 0.500000 1.0 0.986973 \n", + "2 1.000000 0.8 0.981254 \n", + "3 0.666667 1.0 0.979251 \n", + "4 0.750000 1.0 0.965414 " + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.reset_option('display.max_colwidth')\n", + "pd.reset_option('display.max_rows')\n", + "\n", + "data = pd.read_csv('evaluation_results.csv')\n", + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "667df35e-e61e-41a4-bb1a-b59ce9656f78", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
user_inputresponse
0What is the policy regarding smoking in City offices?According to the document, smoking is not permitted in City offices, or within n 20 feet of entrances, exits, or operable windows of public buildings.
1How many days of paid sick leave do most full-time employees earn per year under Civil Service Rules?Most full-time employees earn 13 8-hour working days per year of paid sick leave under the Civil Service Rules.
2What are the three categories of employees eligible for health coverage?The three categories of employees eligible for health coverage are:\\n\\n1. All permanent employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours.\\n2. All regularly scheduled provisional employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours.\\n3. All other employees of the City, including temporary, exempt, or \"as needed\" employees who have worked more than 1040 hours in any consecutive 12-month period, and whose normal work week at the time of inclusion in the system is not less than 20 hours.
3How long must an employee wait before using vacation time after starting employment?An employee must wait at least one year of continuous service before using vacation time. At the end of the first year, the employee will be awarded a vacation allowance at the rate of 0.0385 of an hour for each hour of paid service in the preceding year, and will be eligible to use accrued vacation time.
4What must an employee do if they're summoned for jury duty?An employee must notify their supervisor as soon as they receive a jury summons. If they are required to report for jury duty during their working hours, they will be excused from work on the work day they perform jury service, provided they give prior notification to their supervisor.
\n", + "
" + ], + "text/plain": [ + " user_input \\\n", + "0 What is the policy regarding smoking in City offices? \n", + "1 How many days of paid sick leave do most full-time employees earn per year under Civil Service Rules? \n", + "2 What are the three categories of employees eligible for health coverage? \n", + "3 How long must an employee wait before using vacation time after starting employment? \n", + "4 What must an employee do if they're summoned for jury duty? \n", + "\n", + " response \n", + "0 According to the document, smoking is not permitted in City offices, or within n 20 feet of entrances, exits, or operable windows of public buildings. \n", + "1 Most full-time employees earn 13 8-hour working days per year of paid sick leave under the Civil Service Rules. \n", + "2 The three categories of employees eligible for health coverage are:\\n\\n1. All permanent employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours.\\n2. All regularly scheduled provisional employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours.\\n3. All other employees of the City, including temporary, exempt, or \"as needed\" employees who have worked more than 1040 hours in any consecutive 12-month period, and whose normal work week at the time of inclusion in the system is not less than 20 hours. \n", + "3 An employee must wait at least one year of continuous service before using vacation time. At the end of the first year, the employee will be awarded a vacation allowance at the rate of 0.0385 of an hour for each hour of paid service in the preceding year, and will be eligible to use accrued vacation time. \n", + "4 An employee must notify their supervisor as soon as they receive a jury summons. If they are required to report for jury duty during their working hours, they will be excused from work on the work day they perform jury service, provided they give prior notification to their supervisor. " + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.set_option('display.max_colwidth', None)\n", + "pd.set_option('display.max_rows', None)\n", + "\n", + "data[['user_input', 'response']].head()" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "df53125e-70e1-401c-85b9-10e3c668ba08", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
retrieved_contexts
0['## Smoke-Free Workplace\\n\\nSmoking is not permitted in City offices, or within n 20 feet of entrances, exits, or operable windows of public buildings.\\n\\n## Drug-Free Workplace', '## Policy Regarding the Treatment of Co-Workers and Members of the Public\\n\\nCity policy requires employees to treat co-workers and members of the public with courtesy and respect. City employees and managers are responsible for maintaining a safe and productive workplace which is free from inappropriate workplace behavior.\\n\\n## Smoke-Free Workplace', '## Political Activity\\n\\nIt is unlawful for City employees to use public resources or personnel to engage in political activity relating to elective offices and ballot measures. City employees may not engage in political activities while on duty or in the workplace. Employees may not use City resources, such as photocopier or fax machines, telephones, postage, or email, for political activities. The ban on engaging in political activity while on duty prohibits such activities as circulating petitions, addressing campaign mailers or engaging in any other political activities that use City resources or divert employees from their assigned duties.', '## Use of City and County Property for Business Purposes Only\\n\\nNo officer or employee may use, nor allow any other r person to use, City resources for any non-City business purpose. Use of City resources fofor personal, political, employee organization or other non-City business is strictly prohibited. City resources include, but are not limited to, facilities, equipment, devices, telephones, computers, copier, fax machine, email, internet access, supplies and any time for which you are receiving compensation from the City. Inappropriate uses of City resources include, but are not limited to: online gambling; viewing sports events online; playing games, streaming video or music on a work computer; viewing or distributing materials that are not related to City business or that are sexually explicit; and frequent talking on a personal cell phone or texting during work hours.', \"The safety and well-being of our employees is very important, and in order to promote a safe and healthy work environment, the City works to identify and eliminate employee exposures to avoidable hazards and conditions that can lead to injury or illness. City departments have Injury and Illness Prevention Programs that comply with federal and state regulations, laws, and statutes in order to help maintain a safe and healthful work environment.\\n\\nSafety is every City employee's responsibility. All employees are required to remain alert and to correct hazardous conditions and unsafe acts-if it is safe to do so-and to report unsafe conditions to their supervisors.\"]
1['Most full time employees earn 13 8-hour working days per year of paid sick leave under the Civil Service Rules. Unused sick leave accrued under the Civil Service Rules may be accumulated from year to year up to a maximum of 130 8-hour working days.\\n\\nSome employees may be entitled to accrue sick leave under Chapter 12W of the Administrative Code. Please contact your departmental personnel officer for more information.', 'Sick leave with pay is a privilege under the Civil Service Rules, Charter and City Ordinance.\\n\\nMost full time employees earn 13 8-hour working days per year of paid sick leave under the Civil Service Rules. Unused sick leave accrued under the Civil Service Rules may be accumulated from year to year up to a maximum of 130 8-hour working days.', '## Accrual of Paid Sick Leave\\n\\nSick leave with pay is a privilege under the Civil Service Rules, Charter and City Ordinance.', 'The maximum vacation entitlement in any 12 month period and maximum accrual permitted for most employees are provided in the chart on the following page.\\n\\n| Year of Service 12-Month Max. Entitlement | Maximum Accumulation | Maximum Accumulation |\\n|-----------------------------------------------------------|-----------------------------------------------------------|------------------------|\\n| | 1 through 5 years 80 hours (10 days) 320 hours (40 days) | |\\n| More than 5 years 120 hours (15 days) 360 hours (45 days) | | |\\n| More than 15 years 160 hours (20 days) 400 hours | | (50 days) |', 'You may not use sick leave with pay accrued under the Civil Service Rules during your first 6 months of employment with the City.\\n\\nYou may not use sick leave for reasons not set forth th in this Handbook, the Civil Service Rules, the applicable collective bargaining agreement or other applicable laws. Misuse of sick leave is grounds for discipline, up to and including termination.']
2['The following employees are eligible for health coverage:\\n\\n- (1) All permanent employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours;\\n- (2) All regularly scheduled provisional employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours; and\\n- (3) All other employees of the City including temporary y exempt or \"as needed\" employees who have worked more than 1040 hours in any consecutive 12 month period, and whose normal work week at the time of inclusion in the system is not less than 20 hours.', \"Employees in category (3) must obtain a signed certification from their department's human resources manager in order to be eligible for health coverage.\\n\\nAn employee's spouse, domestic partner and children may also be eligible for coverage through HSS. If you have enrolled a domestic partner, same sex spouse and/or their children in your health plan, it is important that you seek tax advice from a qualified professional regarding the tax consequences of such enrollment. A detailed list of eligibility requirements and necessary documentation for enrolling employee dependents is available on the HSS website at www.myhss.org/benefits/ccsf.html .\", 'Please note that the information in this section is subject to change. You should consult with HSS if you have any questions at (415) 554-1750.\\n\\n## Eligibility\\n\\nThe following employees are eligible for health coverage:', \"Coverage for a new employee in category (1) or (2) listed above starts on the first day of the coverage period following his or her start work date, provided an enrollment application and other required documentation has been submitted to HSS by applicable deadlines.\\n\\nEmployees in category (3) must obtain a signed certification from their department's human resources manager in order to be eligible for health coverage.\", '- (1) All permanent employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours;\\n- (2) All regularly scheduled provisional employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours; and\\n- (3) All other employees of the City including temporary y exempt or \"as needed\" employees who have worked more than 1040 hours in any consecutive 12 month period, and whose normal work week at the time of inclusion in the system is not less than 20 hours.\\n\\nCoverage for a new employee in category (1) or (2) listed above starts on the first day of the coverage period following his or her start work date, provided an enrollment application and other required documentation has been submitted to HSS by applicable deadlines.']
3['An additional vacation entitlement will be awarded after five years of continuous service, usually 40 hours for full-time employees. After fifteen years of service, full-time employees will receive an additional 40 hours.\\n\\nThe maximum vacation entitlement in any 12 month period and maximum accrual permitted for most employees are provided in the chart on the following page.', 'The maximum vacation entitlement in any 12 month period and maximum accrual permitted for most employees are provided in the chart on the following page.\\n\\n| Year of Service 12-Month Max. Entitlement | Maximum Accumulation | Maximum Accumulation |\\n|-----------------------------------------------------------|-----------------------------------------------------------|------------------------|\\n| | 1 through 5 years 80 hours (10 days) 320 hours (40 days) | |\\n| More than 5 years 120 hours (15 days) 360 hours (45 days) | | |\\n| More than 15 years 160 hours (20 days) 400 hours | | (50 days) |', 'The City provides annual vacations for employees who work a regular schedule and who have completed one year or more of continuous service. Certain temporary exempt employees may also be eligible for vacation benefits. You are not eligible to use vacation in the first year of continuous service; however, at the end of one year of continuous service, you will be awarded a vacation allowance at the rate of .0385 of an hour for each hour of paid service in the preceding year and will be eligible to use accrued vacation time. For the purpose of computing vacation, most employees may be credited with no more than 2080 hours of service in a 12month period.\\n\\nAn additional vacation entitlement will be awarded after five years of continuous service, usually 40 hours for full-time employees. After fifteen years of service, full-time employees will receive an additional 40 hours.', '## Vacation\\n\\nThe City provides annual vacations for employees who work a regular schedule and who have completed one year or more of continuous service. Certain temporary exempt employees may also be eligible for vacation benefits. You are not eligible to use vacation in the first year of continuous service; however, at the end of one year of continuous service, you will be awarded a vacation allowance at the rate of .0385 of an hour for each hour of paid service in the preceding year and will be eligible to use accrued vacation time. For the purpose of computing vacation, most employees may be credited with no more than 2080 hours of service in a 12month period.', 'Permanent employees may request unpaid personal leave for a period of up to twelve (12) months within any two-year period. Your department t head has discretion to grant or deny requests for personal leave. With certain exceptions, if you are a temporary or provisional employee, you may request personal leave for a maximum of one month, and only if a replacement for your position is not required.\\n\\n## Time Off for Voting']
4['You must notify your supervisor as soon as you receive a jury summons. If you are required to report for jury duty during your working hours, you will be excused from work on the work day you perform jury service, provided you give prior notification to your supervisor. If you report for jury duty and are not selected as a juror, or if the court dismisses the proceedings early for the day, you must return to work as soon as possible.\\n\\n## Work Schedule While on Jury Duty', '## Jury Duty Leave\\n\\nYou must notify your supervisor as soon as you receive a jury summons. If you are required to report for jury duty during your working hours, you will be excused from work on the work day you perform jury service, provided you give prior notification to your supervisor. If you report for jury duty and are not selected as a juror, or if the court dismisses the proceedings early for the day, you must return to work as soon as possible.', '## Compensation While on Jury Duty\\n\\nAs a guiding principal, you are required to work or r perform jury service for the number of hours for which you are paid during that workweek. For example, if you are regularly on an alternative workweek schedule working 10-hour workdays Monday through Thursday with Friday off and you are required to serve jury duty on a Monday through Thursday, you must work a regular workday on Friday, or use personal leave to compensate for the eight hour balance. However, you and your supervisor may agree to maintain the alternative schedule, whereby you would return to the workplace to work the remaining two hours each day and continue to take Friday off.', 'As a guiding principal, you are required to work or r perform jury service for the number of hours for which you are paid during that workweek. For example, if you are regularly on an alternative workweek schedule working 10-hour workdays Monday through Thursday with Friday off and you are required to serve jury duty on a Monday through Thursday, you must work a regular workday on Friday, or use personal leave to compensate for the eight hour balance. However, you and your supervisor may agree to maintain the alternative schedule, whereby you would return to the workplace to work the remaining two hours each day and continue to take Friday off.\\n\\nYou are not entitled to a per diem pay from the City or County for which Jury Duty was served, as the City and County of San Francisco already provides regular compensation and benefits to you while performing jury service. You must notify the jury staff that you are a government employee when reporting for jury service', 'If you are summoned as a witness on behalf of the City and County of San Francisco, you are entitled to be paid for any time that you are required to serve in that capacity. If you are summoned to serve as a witness in a case involving outside employment or personal business\\n\\naffairs, you will be placed on leave without pay unless vacation leave or compensatory time is requested and granted.']
\n", + "
" + ], + "text/plain": [ + " retrieved_contexts\n", + "0 ['## Smoke-Free Workplace\\n\\nSmoking is not permitted in City offices, or within n 20 feet of entrances, exits, or operable windows of public buildings.\\n\\n## Drug-Free Workplace', '## Policy Regarding the Treatment of Co-Workers and Members of the Public\\n\\nCity policy requires employees to treat co-workers and members of the public with courtesy and respect. City employees and managers are responsible for maintaining a safe and productive workplace which is free from inappropriate workplace behavior.\\n\\n## Smoke-Free Workplace', '## Political Activity\\n\\nIt is unlawful for City employees to use public resources or personnel to engage in political activity relating to elective offices and ballot measures. City employees may not engage in political activities while on duty or in the workplace. Employees may not use City resources, such as photocopier or fax machines, telephones, postage, or email, for political activities. The ban on engaging in political activity while on duty prohibits such activities as circulating petitions, addressing campaign mailers or engaging in any other political activities that use City resources or divert employees from their assigned duties.', '## Use of City and County Property for Business Purposes Only\\n\\nNo officer or employee may use, nor allow any other r person to use, City resources for any non-City business purpose. Use of City resources fofor personal, political, employee organization or other non-City business is strictly prohibited. City resources include, but are not limited to, facilities, equipment, devices, telephones, computers, copier, fax machine, email, internet access, supplies and any time for which you are receiving compensation from the City. Inappropriate uses of City resources include, but are not limited to: online gambling; viewing sports events online; playing games, streaming video or music on a work computer; viewing or distributing materials that are not related to City business or that are sexually explicit; and frequent talking on a personal cell phone or texting during work hours.', \"The safety and well-being of our employees is very important, and in order to promote a safe and healthy work environment, the City works to identify and eliminate employee exposures to avoidable hazards and conditions that can lead to injury or illness. City departments have Injury and Illness Prevention Programs that comply with federal and state regulations, laws, and statutes in order to help maintain a safe and healthful work environment.\\n\\nSafety is every City employee's responsibility. All employees are required to remain alert and to correct hazardous conditions and unsafe acts-if it is safe to do so-and to report unsafe conditions to their supervisors.\"]\n", + "1 ['Most full time employees earn 13 8-hour working days per year of paid sick leave under the Civil Service Rules. Unused sick leave accrued under the Civil Service Rules may be accumulated from year to year up to a maximum of 130 8-hour working days.\\n\\nSome employees may be entitled to accrue sick leave under Chapter 12W of the Administrative Code. Please contact your departmental personnel officer for more information.', 'Sick leave with pay is a privilege under the Civil Service Rules, Charter and City Ordinance.\\n\\nMost full time employees earn 13 8-hour working days per year of paid sick leave under the Civil Service Rules. Unused sick leave accrued under the Civil Service Rules may be accumulated from year to year up to a maximum of 130 8-hour working days.', '## Accrual of Paid Sick Leave\\n\\nSick leave with pay is a privilege under the Civil Service Rules, Charter and City Ordinance.', 'The maximum vacation entitlement in any 12 month period and maximum accrual permitted for most employees are provided in the chart on the following page.\\n\\n| Year of Service 12-Month Max. Entitlement | Maximum Accumulation | Maximum Accumulation |\\n|-----------------------------------------------------------|-----------------------------------------------------------|------------------------|\\n| | 1 through 5 years 80 hours (10 days) 320 hours (40 days) | |\\n| More than 5 years 120 hours (15 days) 360 hours (45 days) | | |\\n| More than 15 years 160 hours (20 days) 400 hours | | (50 days) |', 'You may not use sick leave with pay accrued under the Civil Service Rules during your first 6 months of employment with the City.\\n\\nYou may not use sick leave for reasons not set forth th in this Handbook, the Civil Service Rules, the applicable collective bargaining agreement or other applicable laws. Misuse of sick leave is grounds for discipline, up to and including termination.']\n", + "2 ['The following employees are eligible for health coverage:\\n\\n- (1) All permanent employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours;\\n- (2) All regularly scheduled provisional employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours; and\\n- (3) All other employees of the City including temporary y exempt or \"as needed\" employees who have worked more than 1040 hours in any consecutive 12 month period, and whose normal work week at the time of inclusion in the system is not less than 20 hours.', \"Employees in category (3) must obtain a signed certification from their department's human resources manager in order to be eligible for health coverage.\\n\\nAn employee's spouse, domestic partner and children may also be eligible for coverage through HSS. If you have enrolled a domestic partner, same sex spouse and/or their children in your health plan, it is important that you seek tax advice from a qualified professional regarding the tax consequences of such enrollment. A detailed list of eligibility requirements and necessary documentation for enrolling employee dependents is available on the HSS website at www.myhss.org/benefits/ccsf.html .\", 'Please note that the information in this section is subject to change. You should consult with HSS if you have any questions at (415) 554-1750.\\n\\n## Eligibility\\n\\nThe following employees are eligible for health coverage:', \"Coverage for a new employee in category (1) or (2) listed above starts on the first day of the coverage period following his or her start work date, provided an enrollment application and other required documentation has been submitted to HSS by applicable deadlines.\\n\\nEmployees in category (3) must obtain a signed certification from their department's human resources manager in order to be eligible for health coverage.\", '- (1) All permanent employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours;\\n- (2) All regularly scheduled provisional employees of the City whose normal work week at the time of inclusion in the system is not less than 20 hours; and\\n- (3) All other employees of the City including temporary y exempt or \"as needed\" employees who have worked more than 1040 hours in any consecutive 12 month period, and whose normal work week at the time of inclusion in the system is not less than 20 hours.\\n\\nCoverage for a new employee in category (1) or (2) listed above starts on the first day of the coverage period following his or her start work date, provided an enrollment application and other required documentation has been submitted to HSS by applicable deadlines.']\n", + "3 ['An additional vacation entitlement will be awarded after five years of continuous service, usually 40 hours for full-time employees. After fifteen years of service, full-time employees will receive an additional 40 hours.\\n\\nThe maximum vacation entitlement in any 12 month period and maximum accrual permitted for most employees are provided in the chart on the following page.', 'The maximum vacation entitlement in any 12 month period and maximum accrual permitted for most employees are provided in the chart on the following page.\\n\\n| Year of Service 12-Month Max. Entitlement | Maximum Accumulation | Maximum Accumulation |\\n|-----------------------------------------------------------|-----------------------------------------------------------|------------------------|\\n| | 1 through 5 years 80 hours (10 days) 320 hours (40 days) | |\\n| More than 5 years 120 hours (15 days) 360 hours (45 days) | | |\\n| More than 15 years 160 hours (20 days) 400 hours | | (50 days) |', 'The City provides annual vacations for employees who work a regular schedule and who have completed one year or more of continuous service. Certain temporary exempt employees may also be eligible for vacation benefits. You are not eligible to use vacation in the first year of continuous service; however, at the end of one year of continuous service, you will be awarded a vacation allowance at the rate of .0385 of an hour for each hour of paid service in the preceding year and will be eligible to use accrued vacation time. For the purpose of computing vacation, most employees may be credited with no more than 2080 hours of service in a 12month period.\\n\\nAn additional vacation entitlement will be awarded after five years of continuous service, usually 40 hours for full-time employees. After fifteen years of service, full-time employees will receive an additional 40 hours.', '## Vacation\\n\\nThe City provides annual vacations for employees who work a regular schedule and who have completed one year or more of continuous service. Certain temporary exempt employees may also be eligible for vacation benefits. You are not eligible to use vacation in the first year of continuous service; however, at the end of one year of continuous service, you will be awarded a vacation allowance at the rate of .0385 of an hour for each hour of paid service in the preceding year and will be eligible to use accrued vacation time. For the purpose of computing vacation, most employees may be credited with no more than 2080 hours of service in a 12month period.', 'Permanent employees may request unpaid personal leave for a period of up to twelve (12) months within any two-year period. Your department t head has discretion to grant or deny requests for personal leave. With certain exceptions, if you are a temporary or provisional employee, you may request personal leave for a maximum of one month, and only if a replacement for your position is not required.\\n\\n## Time Off for Voting']\n", + "4 ['You must notify your supervisor as soon as you receive a jury summons. If you are required to report for jury duty during your working hours, you will be excused from work on the work day you perform jury service, provided you give prior notification to your supervisor. If you report for jury duty and are not selected as a juror, or if the court dismisses the proceedings early for the day, you must return to work as soon as possible.\\n\\n## Work Schedule While on Jury Duty', '## Jury Duty Leave\\n\\nYou must notify your supervisor as soon as you receive a jury summons. If you are required to report for jury duty during your working hours, you will be excused from work on the work day you perform jury service, provided you give prior notification to your supervisor. If you report for jury duty and are not selected as a juror, or if the court dismisses the proceedings early for the day, you must return to work as soon as possible.', '## Compensation While on Jury Duty\\n\\nAs a guiding principal, you are required to work or r perform jury service for the number of hours for which you are paid during that workweek. For example, if you are regularly on an alternative workweek schedule working 10-hour workdays Monday through Thursday with Friday off and you are required to serve jury duty on a Monday through Thursday, you must work a regular workday on Friday, or use personal leave to compensate for the eight hour balance. However, you and your supervisor may agree to maintain the alternative schedule, whereby you would return to the workplace to work the remaining two hours each day and continue to take Friday off.', 'As a guiding principal, you are required to work or r perform jury service for the number of hours for which you are paid during that workweek. For example, if you are regularly on an alternative workweek schedule working 10-hour workdays Monday through Thursday with Friday off and you are required to serve jury duty on a Monday through Thursday, you must work a regular workday on Friday, or use personal leave to compensate for the eight hour balance. However, you and your supervisor may agree to maintain the alternative schedule, whereby you would return to the workplace to work the remaining two hours each day and continue to take Friday off.\\n\\nYou are not entitled to a per diem pay from the City or County for which Jury Duty was served, as the City and County of San Francisco already provides regular compensation and benefits to you while performing jury service. You must notify the jury staff that you are a government employee when reporting for jury service', 'If you are summoned as a witness on behalf of the City and County of San Francisco, you are entitled to be paid for any time that you are required to serve in that capacity. If you are summoned to serve as a witness in a case involving outside employment or personal business\\n\\naffairs, you will be placed on leave without pay unless vacation leave or compensatory time is requested and granted.']" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data[['retrieved_contexts']].head()" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "#### **4. Visualizing the Metrics**\n", + "\n", + "Let's create bar charts to visualize each metric for the different user inputs." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "def plot_metric(metric):\n", + " data['short_user_input'] = data['user_input'].apply(lambda x: x[:10] + '...' if len(x) > 10 else x)\n", + " plt.figure(figsize=(7, 4))\n", + " sns.barplot(x='short_user_input', y=metric, data=data, hue='short_user_input', palette='viridis', legend=False)\n", + " plt.xticks(rotation=75, ha='right', fontsize=8)\n", + " plt.ylim(0, 1.1)\n", + " plt.title(f'{metric.replace(\"_\", \" \").title()} for Each User Input')\n", + " plt.ylabel(metric.replace(\"_\", \" \").title())\n", + " plt.xlabel('User Input')\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "210298cf-b285-4eea-b980-0190520cc20d", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAHMCAYAAACUdN+cAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABm9UlEQVR4nO3dd1hT5/8+8BsCyHQwVZYTEAERB+Kuo1hX3XvbOj5qrVu/te6KbdWq1bZOqlZFK4Ja0VatOHFWxQUqoqAgsmUHkvz+8JdoBJUocCLnfl2X1yUnJ8k7D8nhzjPO0VEoFAoQERERkaB0hS6AiIiIiBjKiIiIiLQCQxkRERGRFmAoIyIiItICDGVEREREWoChjIiIiEgLMJQRERERaQGGMiIiIiItwFBGREREpAUYyoioTO3btw/Ozs54/PixRvdzdnbGzz//XEpVvd2pU6fw+eefw93dHc7Oznj+/LkgdZSmdu3aYezYsUKXQSRqDGVU7sTExGDevHlo37493N3d4eXlhQEDBmDr1q3Izc0ttee9f/8+fv75Z43Dxvv47bffcOzYsWLt+/jxYzg7O6v+1atXD23btsWECRNw586dUq7045eamoqvv/4ahoaGmDdvHn744QcYGRmV2vMpQ+ub/l27dq3Unvt9Kd9jmzdvLvL2zZs3v1cQLy2zZ89Gw4YNhS5DpSyPHaTd9IQugKgkhYaGYvLkyTAwMMDnn38OJycn5Ofn48qVK/jxxx9x//59LF68uFSe+/79+1i7di2aNm0KOzu7UnkOpfXr18PX1xcdOnQo9n26du2K1q1bQy6XIyoqCrt27cKpU6ewZ88e1KtXrxSrVff555+jS5cuMDAw0Oh+4eHhkEgkpVTVm924cQNZWVmYPHkymjdvXmbP+9VXXxX5PnJwcCizGqhslOWxg7QbQxmVG7GxsZgyZQqqV6+OrVu3wtraWnXb4MGD8ejRI4SGhgpXoMBcXV3x+eefq3728vLC+PHjsWvXLixatKjI+2RnZ8PY2LhE65BIJO8VripUqFCidRRXSkoKAMDMzKzEHrM47dq6dWu4u7uX2HOKiUKhQF5eHgwNDYUuhUgjHL6kcmPTpk3Izs7Gd999pxbIlBwdHTF8+HDVzwUFBVi3bh06dOgANzc3tGvXDitXroRUKlW7n3KuzeXLl9GnTx+4u7ujffv2CA4OVu2zb98+TJ48GQAwbNgw1VDThQsXVPucPHkSgwYNgqenJxo2bIgxY8bg3r17qtvDwsLg4uKC1atXqz3/wYMH4ezsjJ07dwJ4MbcqOzsbQUFBqueZPXu2xu3VrFkzAFANmSiHzS5evIgFCxbAx8cHbdq0KXb9SlFRUZg8eTKaNWsGDw8P+Pr64qefflJrq9eHsm7cuIHRo0fD29sbHh4eaNeuHebMmaP2uEXNKbt9+za++OILeHl5oWHDhhg+fHih4T3l8125cgV+fn5o1qwZPD09MWHCBFXgepOhQ4di1qxZAIA+ffoUauvDhw+jV69e8PDwgLe3N6ZPn46EhAS1x1AOlcXExODLL79Ew4YNMX369Lc+b3Ft3rwZAwYMULVbr169cOTIkSL33b9/P/r06YMGDRqgSZMmGDx4MM6cOVNov7e9z0tScX7ncrkcv//+O7p06QJ3d3c0b94c8+bNQ3p6utp+ys/o6dOnVb+PgIAAjeopzuccePl+unTpEubNmwdvb294eXlh5syZhep60zzIdu3aqd5HxTl2kHiwp4zKjRMnTsDe3h5eXl7F2n/u3LkICgqCr68vRo4cifDwcKxfvx5RUVFYt26d2r6PHj3C5MmT0adPH/Ts2ROBgYGYPXs26tevj7p166JJkyYYOnQotm/fjnHjxqFWrVoAgNq1awMAgoODMXv2bLRs2RLTp09HTk4Odu3ahUGDBiEoKAh2dnbw8fHBoEGDsGHDBnTo0AH169fHs2fPsGTJEjRv3hwDBw4EAPzwww+YO3cuPDw80K9fPwDvN6QVExMDAKhcubLa9oULF8Lc3BwTJkxAdnZ2sesHgIiICAwePBh6enro378/bG1tERMTg3///RdTpkwpso7k5GSMHj0aVapUwZgxY1CxYkU8fvwYR48efWv99+7dw+DBg2FiYoIvvvgCenp62L17N4YOHYo//vgDDRo0UNt/yZIlqFixIiZOnIgnT55g69atWLRoEVatWvXG5xg3bhxq1qyJ3bt3q4YTlW29b98+zJkzB+7u7pg6dSqSk5Oxbds2/PfffwgODkbFihVVj1NQUIDRo0ejUaNGmDVrVrF6cDIzMwuFRh0dHVSpUkX187Zt29CuXTt069YN+fn5OHToECZPnoz169ejbdu2qv3Wrl2Ln3/+GQ0bNsRXX30FfX19XL9+HefPn0fLli1V+73rfV5Sivs7nzdvHoKCgtCrVy8MHToUjx8/xo4dO3D79m3s2rUL+vr6qn2jo6Mxbdo09O/fH/369UPNmjU1rkuT179o0SLV+yk6Ohq7du1CXFwctm/fDh0dnWI/57uOHSQyCqJyICMjQ+Hk5KQYP358sfa/c+eOwsnJSfHNN9+obV+2bJnCyclJERYWptr2ySefKJycnBSXLl1SbUtOTla4ubkpli1bptp2+PBhhZOTk+L8+fNqj5mZmalo3LixYu7cuWrbExMTFY0aNVLbnp2drejYsaOiS5cuiry8PMWYMWMUXl5eiidPnqjd19PTUzFr1qxivdbY2FiFk5OT4ueff1YkJycrEhMTFRcuXFD06NFD4eTkpPj7778VCoVCERgYqHByclIMHDhQUVBQ8F71Dx48WNGwYcNC9crlctX/lc8TGxurUCgUiqNHjyqcnJwU4eHhb30dTk5OijVr1qh+/t///qeoX7++IiYmRrUtISFB0bBhQ8XgwYMLPd+IESPU6li6dKmiXr16iufPn7/1eZX3f7U+qVSq8PHxUXTt2lWRm5ur2n7ixAmFk5OTYvXq1apts2bNUjg5OSmWL1/+1ud5/fmK+ufm5qa2b05OjtrPUqlU0bVrV8WwYcNU2x4+fKhwcXFRTJgwQSGTydT2f7U9ivs+L4ryPbZp06Yib9+0aZPGv/NLly4pnJycFAcOHFDbfurUqULblbWfOnXqrXUqzZo1S+Hp6am2rbivX/n76dmzp0Iqlaq2b9y4UeHk5KQ4duyYatvr79lXn+vVz++bjh0kPhy+pHIhMzMTAGBiYlKs/U+ePAkAGDlypNr2UaNGqd2uVKdOHTRu3Fj1s7m5OWrWrInY2Nh3Pte5c+fw/PlzdOnSBSkpKap/urq6aNCggdowhZGREfz8/BAVFYXBgwcjNDQUc+bMQfXq1Yv1ut7m559/ho+PD1q0aIGhQ4ciJiYG06dPx6effqq2X79+/dTmfBW3/pSUFFy6dAm9e/cuVO/beg6Uc7VCQ0ORn59frNcik8lw9uxZdOjQAfb29qrt1tbW6Nq1K65cuaJ6T7z6ul6to3HjxpDJZHjy5EmxnvNVN2/eRHJyMgYOHKg2161t27aoVatWkXMXlT2dxTVv3jz4+/ur/du4caPaPq/2uKWnpyMjIwONGjXC7du3VduPHTsGuVyOCRMmQFdX/ZD/+u/lQ97nmijO7/zIkSMwMzNDixYt1N539evXh7GxcaHhPTs7O7Rq1eqD6tLk9ffv31+tp27gwIHQ09MrdOwg0gSHL6lcMDU1BQBkZWUVa/8nT55AV1e30LCflZUVKlasWOgPdbVq1Qo9RqVKlQrNISnKw4cPAUBtPturlLUrNWrUCAMHDsSOHTvQsmVL9OnT553PURz9+/dHp06doKOjg4oVK6Ju3bpFroB8ffVXcetX/uFycnLSqK6mTZvC19cXa9euxe+//46mTZuiQ4cO6Nat2xtXaKakpCAnJ6fIIaratWtDLpcjPj5ebcjp9aCoHF58n3OOxcXFAUCRz1+rVi1cuXJFbZuenh6qVq2q0XN4eHi8c6L/iRMn8Ouvv+LOnTtqcyFfDVsxMTHQ1dUt1nDYh7zPi0NZV3F+548ePUJGRgZ8fHyKfKzk5GS1n0ti1aImr9/R0VHtZxMTE1hZWb1XyCdSYiijcsHU1BTW1tZFTjx/m+LO/fiQUzEoFAoAL+aCWVlZvfOxpVIpLl68COBF0MnJySmR82I5OjoW65QOr69y1LR+Teno6GDNmjW4du0aTpw4gdOnT+P//u//4O/vj927dxe79/NdXu8lUlK+vtJkYGDwxud/X5cvX8b48ePRpEkTzJ8/H1ZWVtDX10dgYCD++uuv93rM9/1dKt8zbzoPYE5Ojtp+xfmdy+VyWFhYYPny5UU+prm5udrPJbHSsqxOuSKTycrkeejjw1BG5cYnn3yC3bt34+rVq+88MaStrS3kcjkePXqk1oOQlJSE58+fw9bWVuPnf1PAUw6vWVhYFCsUrVmzBlFRUZg1axaWL1+OFStWYO7cuRrXU1KKW79yv7t3777X83h6esLT0xNTpkzBwYMHMX36dISEhKBv376F9jU3N4eRkRGio6ML3fbgwQPo6uoW2etRUpS9btHR0YV6cqKjo0tkuPld/v77b1SoUAGbN29W61EMDAxU28/BwUF1brrSOh/d234fwIs2MTIyUlukALz9d+7g4ICwsDB4eXlp5aktHj16pFrBDLzopU9MTETr1q1V2ypVqlSoJ1YqlSIxMVFtmyYLA6h845wyKje++OILGBsbY+7cuUhKSip0e0xMDLZu3QoAqlM9KH9W8vf3V7tdE8rerIyMDLXtrVq1gqmpKdavX1/k/JlXV9hdv34dW7ZswfDhwzFq1CiMHj0af/zxh6rnTMnY2LjMLvVT3PrNzc3RpEkTBAYGqob3lN7WG5Wenl7odmV4eP30JEoSiQQtWrTA8ePH1U6tkZSUhL/++guNGjUqNCxcktzc3GBhYYGAgAC1Gk+ePImoqCi1lY+lRSKRQEdHR63X5fHjxzh+/Ljafh06dICuri7WrVsHuVyudltJ9RIqfx8nTpwo9LuPi4vDiRMn0KJFC1VPVHF+55999hlkMhl++eWXQs9XUFAg+KWudu/erfZ52LVrFwoKCtRCmb29PS5fvqx2vz179hTqKXvTsYPEhz1lVG44ODhg+fLlmDJlCjp37qw6o79UKsXVq1dx5MgR9OrVCwDg4uKCnj17Yvfu3Xj+/DmaNGmCGzduICgoCB06dFD7Blxc9erVg0QiwcaNG5GRkQEDAwM0a9YMFhYWWLBgAWbOnIlevXqhc+fOMDc3R1xcHE6ePAkvLy/MmzcPeXl5mDVrFhwdHVWnj5g0aRJOnDiBOXPm4ODBg6oTjtavXx9hYWHw9/eHtbU17OzsCp0CoqSYmpoWq37gxWlGBg4ciJ49e6J///6ws7PDkydPEBoaiv379xf5+EFBQdi1axc6dOgABwcHZGVlYc+ePTA1NVX7A/e6r7/+GufOncOgQYMwaNAgSCQS7N69G1KpFDNmzCiVtlDS19fH9OnTMWfOHAwZMgRdunRRnRLD1tYWI0aM+ODnOHXqFB48eFBou5eXF+zt7dGmTRv4+/vjiy++QNeuXZGcnIydO3fCwcEBkZGRqv0dHR0xbtw4/PLLLxg0aBA+/fRTGBgY4MaNG7C2tsa0adM+uFYAmDp1Kvr166f63dva2uLJkyfYvXs3dHR0MHXqVNW+xfmdN23aFP3798f69etx584dtGjRAvr6+nj48CGOHDmCb775Bp06dSqR2t9Hfn4+RowYgc8++wzR0dHYuXMnGjVqhPbt26v26du3L+bPn49JkyahefPmiIiIwJkzZwr1GL7t2EHiwlBG5Ur79u1x4MABbN68GcePH8euXbtgYGCgOumn8rxewIvzVtnZ2SEoKAjHjh2DpaUlxo4di4kTJ77Xc1tZWWHhwoVYv349vvnmG8hkMmzbtg0WFhbo1q0brK2tsWHDBmzevBlSqRQ2NjZo3LixKiiuXLkSMTExCAgIUM29MTAwwLJly9C/f3/88MMPWLBgAYAXJySdN28eVq1ahdzcXPTs2bPUQhmAYtUPvAi7e/bswerVq7Fr1y7k5eWhevXq+Oyzz9742E2bNsWNGzcQEhKCpKQkmJmZwcPDA8uXL1dbWfm6unXrYseOHVixYgXWr18PhUIBDw8P/Pjjj6XaFkq9evWCoaEhNm7ciOXLl8PY2BgdOnTAjBkz1M5R9r7WrFlT5HY/Pz/Y29vDx8cH3333HTZu3IilS5fCzs4O06dPx5MnT9RCGQBMnjwZdnZ2+OOPP/DTTz/ByMgIzs7Oald4+FC1a9fGnj17sHbtWuzduxfp6emoVKkSWrRogQkTJqhNEyju73zRokVwc3NDQEAAfvrpJ0gkEtja2qJ79+7FPh9haZk3bx4OHjyINWvWID8/H126dMHcuXPVhiL79euHx48fY+/evTh9+jQaNWoEf3//QqH9bccOEhcdRVnMciUiIioHlCcN3rt3Ly+DRSWOc8qIiIiItABDGREREZEWYCgjIiIi0gKcU0ZERESkBdhTRkRERKQFGMqIiIiItIDozlMml8tRUFAAXV1dXtqCiIiISp1CoYBcLoeent5br4MrulBWUFCAGzduCF0GERERiYy7u7vatWpfJ7pQpkyo7u7uquuwEREREZUWmUyGGzduvLWXDBBhKFMOWUokEoYyIiIiKjPvmjbFif5EREREWoChjIiIiEgLMJQRERERaQGGMiIiIiItwFBGREREpAUYyoiIiIi0AEMZERERkRZgKCMiIiLSAgxlRERERFqAoYyIiIhICzCUEREREWkBhjIiIiIiLcBQRkRERKQFGMqIiIiItABDGREREZEWYCgjIiIi0gIMZURERERagKGMiIiISAswlBERERFpAYYyIiIiIi3AUEZERESkBQQNZZcuXcK4cePQsmVLODs749ixY++8z4ULF9CzZ0+4ubmhY8eO2LdvXxlUSkRERFS6BA1l2dnZcHZ2xvz584u1f2xsLMaOHQtvb2/s378fw4cPx9y5c3H69OlSrpSIiIiodOkJ+eRt2rRBmzZtir1/QEAA7OzsMHv2bABA7dq1ceXKFfz+++9o1apVaZVJREREVOoEDWWaunbtGnx8fNS2tWzZEkuXLtX4sWQyWUmV9V5+/fVX7N+/H59//jnGjx8vaC1ERERUeoqbOT6qUJaUlARLS0u1bZaWlsjMzERubi4MDQ2L/Vg3btwo6fKKTSqVIjg4GAqFAvv370ejRo1gYGAgWD1voq+vD1dXV+jpfVRvk2IpKCjA7du3kZ+fL3QpROWOvr4+XOu7Qk9SDo8dsgLcvqXZsYPtQcVV/t4hxeTu7g6JRCLIc2dmZkKhUAAA5HI5XF1dYWpqKkgt7yKRSPDDT/sQ8zhR6FJKjIOdFWZO6YX69esLXQpRuSWRSLAsbDdinj8TupQS41DRGrN9+r/XsUMikSDg9hY8y44vhcqEYW1cDQNcR/FYWgwymaxYnUEfVSiztLREUlKS2rakpCSYmppq1EsGvPiACBXKXn9eIWspjpjHiYh68FToMkqcNrc5UXkQ8/wZ7qfGCV1GiXvfY8ez7HjEZcaWcDXC47G05HxU5ynz9PTE+fPn1badO3cOnp6ewhREREREVEIEDWVZWVm4c+cO7ty5AwB4/Pgx7ty5g7i4F9+sVqxYgZkzZ6r2HzBgAGJjY/HDDz8gKioKO3bswOHDhzFixAghyiciIiIqMYIOX968eRPDhg1T/ezn5wcA6NmzJ5YtW4bExETEx78cf7e3t8f69evh5+eHbdu2oWrVqliyZAlPh0FEREQfPUFDmbe3NyIjI994+7Jly4q8T3BwcClWRURERFT2Pqo5ZURERETlFUMZERERkRZgKCMiIiLSAgxlRERERFqAoYyIiIhICzCUEREREWkBhjIiIiIiLcBQRkRERKQFGMqIiIiItABD2WvkMrnQJZSK8vq6iIiIygtBL7OkjXQluvhx7jbERieU2nPI5PlqP8/6cg0kuvql9nz2NW0wY8mwd+9IREREgmEoK0JsdAKiIh6X2uMrdGRApZc/R9+Lg45CUmrPR0RERNqPw5dEREREWoChjIiIiEgLMJQRERERaQGGMiIiIiItwFBGRFQCZPLye9qZ8vzaiLQJV18SEZUAia4u5hzeiwcpSUKXUqJqmVvC77M+QpdBJAoMZUREJeRBShIinsULXQYRfaQ4fElERESkBRjKiIiIiLQAQxkRERGRFmAoIyIiItICDGVEREREWoChjIiIiEgLMJQJQaEDKJT///8/ExERkagxlAlAB7qQSC0ABSCRWkCHvwYiIiLR48ljBaKfUw36OdWELoOIiIi0BLtoiIiIiLQAQxkRERGRFmAoIyIiojKlUMiELqFUfOjr4pwyIiIiKlM6OhKcv/cdnufECF1Kialo5IBmdb/5oMdgKCMiIqIy9zwnBmnZ94QuQ6tw+JKIiIhICzCUEREREWkBhjIiIiIiLcBQRkRERKQFGMqIiIiItABDGREREZEWYCgjIiIi0gIMZURERERagKGMiIiISAswlBERERFpAYYyIiIiIi3AUEZERESkBQQPZTt27EC7du3g7u6Ovn37Ijw8/K37//777/D19YWHhwfatGmDpUuXIi8vr4yqJSIiIiodgoaykJAQ+Pn5YcKECQgKCoKLiwtGjx6N5OTkIvc/ePAgVqxYgYkTJyIkJATfffcdQkJCsHLlyjKunIiIiKhkCRrK/P390a9fP/Tu3Rt16tTBwoULYWhoiMDAwCL3v3r1Kry8vNCtWzfY2dmhZcuW6Nq16zt714iIiIi0nZ5QTyyVSnHr1i2MHTtWtU1XVxfNmzfH1atXi7xPw4YNceDAAYSHh8PDwwOxsbE4efIkPv/8c42fXyaTFbldIpFo/Fgfize95rdhexAVT3n+rACaf17Kc3vwWKqO7aGuqPYobhsJFspSU1Mhk8lgYWGhtt3CwgIPHjwo8j7dunVDamoqBg0aBIVCgYKCAgwYMADjxo3T+Plv3LhRaJuRkRFcXV01fqyPRWRkJHJycoq9P9uDqHjK+2cF0OzzUt7bg8dSdWwPdR/yt0WwUPY+Lly4gPXr12P+/Pnw8PBATEwMvvvuO6xbtw4TJkzQ6LHc3d3LdVIvirOzs9AlaBW2B1Hx8fPyEttCHdtDXVHtIZPJiuwMep1goaxKlSqQSCSFJvUnJyfD0tKyyPusXr0a3bt3R9++fQG8eOHZ2dmYN28exo8fD13d4k+Rk0gkogtlYnu978L2ICo+fl5eYluoY3uo+5D2EGyiv4GBAerXr4+wsDDVNrlcjrCwMDRs2LDI++Tm5hYKXsoXr1AoSq9YIiIiolIm6PDlyJEjMWvWLLi5ucHDwwNbt25FTk4OevXqBQCYOXMmbGxsMG3aNADAJ598An9/f7i6uqqGL1evXo1PPvmESZ2IiIg+aoKGss6dOyMlJQVr1qxBYmIi6tWrh02bNqmGL+Pj49V6xsaPHw8dHR2sWrUKCQkJMDc3xyeffIIpU6YI9RKIiIiISoTgE/2HDBmCIUOGFHnb9u3b1X7W09PDxIkTMXHixLIojYiIiKjMCH6ZJSIiIiJiKCMiIiLSCgxlRERERFqAoYyIiIhICzCUEREREWmB91p9GRYWhrCwMCQnJ0Mul6vd5ufnVyKFEREREYmJxqFs7dq1WLduHdzc3GBlZQUdHZ3SqIuIiIhIVDQOZQEBAfDz80OPHj1KoRwiIiIicdJ4Tll+fj68vLxKoxYiIiIi0dI4lPXp0wcHDx4sjVqIiIiIREvj4cu8vDzs2bMHYWFhcHZ2hp6e+kPMmTOnxIojIiIiEguNQ1lkZCRcXFwAAHfv3lW7jZP+iYiIiN6PxqHs9YuEExEREdGH+6CTxz59+hRPnz4tqVqIiIiIREvjnjK5XI5ffvkF/v7+yM7OBgCYmJhg5MiRGD9+PHR1eZEAIiIiIk1pHMp++ukn7N27F9OmTVOdGuPKlStYu3YtpFIppkyZUuJFEhEREZV3GoeyoKAgLFmyBO3bt1dtc3FxgY2NDRYuXMhQRkRERPQeNB5rTE9PR61atQptr1WrFtLT00ukKCIiIiKx0TiUubi4YMeOHYW279ixQ3WqDCIiIiLSjMbDlzNmzMDYsWNx7tw5eHp6AgCuXbuG+Ph4bNy4saTrIyIiIhIFjXvKmjZtiiNHjqBjx47IyMhARkYGOnbsiCNHjqBx48alUSMRERFRuadxTxkA2NjYcEI/iZJMLoekHJ72pby+LiKij0mxQllERAScnJygq6uLiIiIt+7LeWVUnkl0dbFocxAexScJXUqJcaxmiXmjewpdBhGR6BUrlPXo0QNnz56FhYUFevToAR0dHSgUikL76ejo4M6dOyVeJJE2eRSfhLuxvJIFERGVrGKFsuPHj8Pc3Fz1fyIiIiIqWcUKZba2tkX+n4iIiIhKhsYze4OCghAaGqr6+YcffkDjxo0xYMAAPHnypCRrIyIiIhINjUPZb7/9hgoVKgAArl69ih07dmDGjBmoXLky/Pz8SrxAIiIiIjHQ+JQYT58+haOjIwDg2LFj8PX1Rf/+/eHl5YWhQ4eWeIFEREREYqBxT5mxsTHS0tIAAGfPnkXz5s0BABUqVEBeXl6JFkdEREQkFhr3lDVv3hxz585FvXr18PDhQ7Rp0wYAcO/ePS4CICIiInpPGveUzZ8/H56enkhJScGaNWtQpUoVAMCtW7fQpUuXEi+QiIiISAw07imrWLEi5s2bV2j7V199VSIFEREREYkRL7NEREREpAVK5DJLyp95mSUiIiKi98PLLBERERFpAV5miYiIiEgLaLz6cv369di7d2+h7Xv37sWGDRtKpCgiIiIisdE4lO3evRu1atUqtL1u3boICAgokaKIiIiIxEbjUJaYmAgrK6tC283NzZGYmFgiRRERERGJjcahrFq1avjvv/8Kbb9y5Qqsra1LpCgiIiIisdH45LF9+/bF0qVLUVBQgGbNmgEAwsLC8OOPP2LUqFElXiARERGRGGgcyr744gukpaVh4cKFyM/PB/DiYuRffPEFxo4dW+IFEhEREYmBxqFMR0cHM2bMwP/+9z9ERUXB0NAQNWrUgIGBQWnUR0RERCQKGs8pU0pKSkJ6ejocHBxgYGCgdoZ/IiIiItKMxqEsNTUVw4cPh6+vL8aMGaNacfl///d/WLZsmcYF7NixA+3atYO7uzv69u2L8PDwt+7//PlzLFy4EC1btoSbmxt8fX1x8uRJjZ+XiIiISJtoHMr8/Pygp6eH0NBQGBoaqrZ37twZp0+f1uixQkJC4OfnhwkTJiAoKAguLi4YPXo0kpOTi9xfKpVi5MiRePLkCVavXo0jR45g8eLFsLGx0fRlEBEREWkVjeeUnT17Fps3b0bVqlXVtteoUQNxcXEaPZa/vz/69euH3r17AwAWLlyI0NBQBAYGYsyYMYX2DwwMRHp6OgICAqCvrw8AsLOz0/QlEBEREWkdjUNZdna2Wg+ZUlpamkaT/aVSKW7duqW2YlNXVxfNmzfH1atXi7zPv//+C09PTyxatEh1kfSuXbviyy+/hEQi0eh1yGSyIrdr+jgfkze95rdhe6hje9CblOf3BqD5+6M8twePHerYHuqKao/itpHGoaxx48YIDg7G119/rdoml8uxadMmeHt7F/txUlNTIZPJYGFhobbdwsICDx48KPI+sbGxOH/+PLp164YNGzYgJiYGCxcuREFBASZOnKjR67hx40ahbUZGRnB1ddXocT4mkZGRyMnJKfb+bA91bA96k/L+3gA0e3+U9/bgsUMd20PdhxxLNQ5lM2bMwIgRI3Dz5k3k5+fjxx9/xP3795Geno5du3a9VxHFpVAoYGFhgcWLF0MikcDNzQ0JCQnYvHmzxqHM3d29XCf1ojg7OwtdglZhe6hje9Db8P3xEttCHdtDXVHtIZPJiuwMep3GoczJyQl///03/vjjD5iYmCA7OxsdO3bE4MGDNbrMUpUqVSCRSApN6k9OToalpWWR97GysoKenp5amKpVqxYSExMhlUo1Gj6VSCSiC2Vie73vwvZQx/agt+H74yW2hTq2h7oPaQ+NQll+fj6++OILLFy4EOPHj3/vJwUAAwMD1K9fH2FhYejQoQOAF8OgYWFhGDJkSJH38fLywl9//QW5XA5d3RcLRx8+fAgrKyuevJaIiIg+ahqdEkNfXx+RkZEl9uQjR47Enj17EBQUhKioKCxYsAA5OTno1asXAGDmzJlYsWKFav+BAwciLS0N3333HaKjoxEaGor169dj8ODBJVYTERERkRA0Hr7s3r079u7di+nTp3/wk3fu3BkpKSlYs2YNEhMTUa9ePWzatEk1fBkfH6/qEQOAatWqYfPmzfDz80P37t1hY2ODYcOG4csvv/zgWoiIiIiEpHEok8lk2LVrF86dOwc3NzcYGRmp3T5nzhyNHm/IkCFvHK7cvn17oW0NGzbEnj17NHoOIiIiIm2ncSi7e/euailrdHS02m06OjolUxURERGRyGgcyorqvSIiIiKiD6NRKAsJCcHx48eRn58PHx8fDBw4sLTqIiIiIhKVYoeynTt3YtGiRXB0dIShoSGOHj2KmJgYzJo1qzTrIyIiIhKFYp8SY8eOHZg4cSL+/vtv7N+/H8uWLSv1M/gTERERiUWxQ1lsbCx69Oih+rlbt24oKCjAs2fPSqMuIiIiIlEpdiiTSqUwNjZ+eUddXejr6yMvL69UCiMiIiISE40m+q9atUrtvGT5+fn49ddfYWZmptqm6XnKiIiIiEiDUNakSZNC5yVr2LAhYmNjVT/zPGVERERE76fYoYznJyMiIiIqPRpdkJyIiIiISgdDGREREZEWYCgjIiIi0gIMZURERERaQONQFhcXB4VCUWi7QqFAXFxciRRFREREJDYah7L27dsjJSWl0Pa0tDS0b9++RIoiIiIiEhuNQ5lCoSjyfGTZ2dmoUKFCiRRFREREJDbFPk+Zn58fgBcniH39zP4ymQzh4eFwcXEp+QqJiIiIRKDYoez27dsAXvSU3b17F/r6+qrbDAwM4OLiglGjRpV8hUREREQioPEZ/efMmYNvvvkGpqampVYUERERkdhoPKdsxowZbwxkkZGRH1wQERERkRhpHMq6deuG0NDQQts3b96Mvn37lkRNRERERKKjcSgbMWIEJk2ahPnz5yM3NxcJCQkYPnw4Nm3ahBUrVpRGjURERETlXrHnlCl9+eWXaNGiBWbOnInu3bsjPT0dHh4eOHDgAKysrEqjRiIiIqJy770us+Tg4IC6deviyZMnyMzMROfOnRnIiIiIiD6AxqHsypUr6N69Ox49eoQDBw5gwYIFWLx4Mb7++mukp6eXRo1ERERE5Z7GoWz48OHo3Lkzdu/ejdq1a6Nv374IDg5GfHw8unXrVho1EhEREZV7Gs8p27JlC5o2baq2zcHBAbt27cKvv/5aYoURERERiYnGPWXKQPbo0SOcPn0aubm5AF5cfmnChAklWx0RERGRSGjcU5aamoqvv/4aFy5cgI6ODv755x/Y29vj//7v/1C5cmXMmjWrNOokIiIiKtc07inz8/ODnp4eQkNDYWhoqNreuXNnnDp1qkSLIyIiIhILjXvKzp49i82bN6Nq1apq22vUqIG4uLgSK4yIiIhITDTuKcvOzlbrIVNKS0uDgYFBiRRFREREJDYah7LGjRsjODhYbZtcLsemTZvg7e1dUnURERERiYrGw5czZszAiBEjcPPmTeTn5+PHH3/E/fv3kZ6ejl27dpVGjURERETlnsahzMnJCX///Tf++OMPmJiYIDs7Gx07dsTgwYNhbW1dGjUSERERlXsah7K4uDhUq1YN48ePL/K26tWrl0hhRERERGKi8Zyy9u3bIyUlpdD21NRUtG/fvkSKIiIiIhIbjUOZQqGAjo5Ooe3Z2dmoUKFCiRRFREREJDbFHr708/MD8OJySqtWrYKRkZHqNplMhvDwcLi4uJR8hUREREQiUOxQdvv2bQAvesru3r0LfX191W0GBgZwcXHBqFGjSr5CIiIiIhEodijbvn07AGDOnDn45ptvYGpqWmpFEREREYmNxqsvlcOYRERERFRyNA5l2dnZ2LBhA86fP4/k5GTI5XK1248fP15ixRERERGJhcahbO7cubh48SI+//xzWFlZFbkSk4jKP5lcDomuxgu4Pwrl+bURkfbSOJSdOnUK69evR6NGjUqsiB07dmDz5s1ITEyEi4sLvv32W3h4eLzzfocOHcLUqVPRvn17/PLLLyVWDxG9m0RXF3N370P0syShSylRNa0tsaR/L6HLICIR0jiUVaxYEZUrVy6xAkJCQuDn54eFCxeiQYMG2Lp1K0aPHo0jR47AwsLijfd7/Pgxvv/+ezRu3LjEaiEizUQ/S0Jk3FOhyyAiKhc07p+fPHkyVq9ejZycnBIpwN/fH/369UPv3r1Rp04dLFy4EIaGhggMDHzjfWQyGaZPn45JkybB3t6+ROogIiIiEpLGPWX+/v6IiYlB8+bNYWdnBz099YcICgoq9mNJpVLcunULY8eOVW3T1dVF8+bNcfXq1Tfeb926dbCwsEDfvn1x5coVTV8CgBfBrigSieS9Hu9j8KbX/DZsD3Vsj5fKc1sAbI/XsT1e4rFDHdtDXVHtUdw20jiUdejQQdO7vFFqaipkMlmhYUoLCws8ePCgyPtcvnwZe/fuRXBw8Ac9940bNwptMzIygqur6wc9rjaLjIzUqIeT7aGO7fFSeW8LgO3xOrbHSzx2qGN7qNO0PV6lcSibOHHiez1RScjMzMTMmTOxePFimJubf9Bjubu7l+ukXhRnZ2ehS9AqbA91bA91bA91bI+X2Bbq2B7qimoPmUxWZGfQ6zQOZUo3b95EVFQUAKBu3brvlXqrVKkCiUSC5ORkte3JycmwtLQstH9sbCyePHmC8ePHq7Ypz5Pm6uqKI0eOwMHBoVjPLZFIRBfKxPZ634XtoY7toY7toY7t8RLbQh3bQ92HtIfGoSw5ORlTpkzBxYsXUbFiRQDA8+fP4e3tjZ9++kmjHiwDAwPUr18fYWFhqmFRuVyOsLAwDBkypND+tWrVwsGDB9W2rVq1CllZWfjmm29QtWpVTV8OERERkVbQOJQtXrwYWVlZOHToEGrXrg0AuH//PmbNmoUlS5Zg5cqVGj3eyJEjMWvWLLi5ucHDwwNbt25FTk4OevV6cZ6gmTNnwsbGBtOmTUOFChXg5OSkdn9lMHx9OxEREdHHRONQdvr0afj7+6sCGQDUqVMH8+fPx6hRozQuoHPnzkhJScGaNWuQmJiIevXqYdOmTarhy/j4eOjyzNpERERUzmkcyuRyOfT19Qs/kJ5eoetgFteQIUOKHK4EgO3bt7/1vsuWLXuv5yQiIiLSJhp3QTVr1gzfffcdEhISVNsSEhLg5+cHHx+fEi2OiIiISCw07imbN28exo8fj/bt26sm1j99+hR169bFjz/+WOIFEhEREYmBxqGsWrVqCAoKwrlz51QneK1duzaaN29e4sURERERicV7nadMR0cHLVq0QIsWLUq6HiIiIiJRKvacsrCwMHTu3BmZmZmFbsvIyECXLl1w+fLlEi2OiIiISCyKHcq2bt2Kfv36wdTUtNBtZmZm6N+/P/z9/Uu0OCIiIiKxKHYoi4yMRKtWrd54e4sWLXDr1q0SKYqIiIhIbIodypKSkqCn9+YpaHp6ekhJSSmRooiIiIjEptihzMbGBvfu3Xvj7ZGRkbCysiqRooiIiIjEptihrE2bNli9ejXy8vIK3Zabm4uff/4Zn3zySYkWR0RERCQWxT4lxvjx4/HPP//A19cXgwcPRs2aNQEADx48wM6dOyGTyTBu3LhSK5SIiIioPCt2KLO0tERAQAAWLFiAlStXQqFQAHhxzrKWLVti3rx5qouIExEREZFmNDp5rK2tLTZu3Ij09HQ8evQIAODo6IhKlSqVSnFEREREYvFeZ/SvVKkSPDw8SroWIiIiItEq9kR/IiIiIio9DGVEREREWoChjIiIiEgLMJQRERERaQGGMiIiIiItwFBGREREpAUYyoiIiIi0AEMZERERkRZgKCMiIiLSAgxlRERERFqAoYyIiIhICzCUEREREWkBhjIiIiIiLcBQRkRERKQFGMqIiIiItABDGREREZEWYCgjIiIi0gIMZURERERagKGMiIiISAswlBERERFpAYYyIiIiIi3AUEZERESkBRjKiIiIiLQAQxkRERGRFmAoIyIiItICDGVEREREWoChjIiIiEgLMJQRERERaQGGMiIiIiItwFBGREREpAUYyoiIiIi0gFaEsh07dqBdu3Zwd3dH3759ER4e/sZ99+zZg0GDBqFJkyZo0qQJRowY8db9iYiIiD4GgoeykJAQ+Pn5YcKECQgKCoKLiwtGjx6N5OTkIve/cOECunTpgm3btiEgIADVqlXDqFGjkJCQUMaVExEREZUcwUOZv78/+vXrh969e6NOnTpYuHAhDA0NERgYWOT+K1aswODBg1GvXj3Url0bS5YsgVwuR1hYWBlXTkRERFRy9IR8cqlUilu3bmHs2LGqbbq6umjevDmuXr1arMfIyclBQUEBKlWqpNFzy2SyIrdLJBKNHudj8qbX/DZsD3Vsj5fKc1sAbI/XsT1e4rFDHdtDXVHtUdw2EjSUpaamQiaTwcLCQm27hYUFHjx4UKzHWL58OaytrdG8eXONnvvGjRuFthkZGcHV1VWjx/mYREZGIicnp9j7sz3UsT1eKu9tAbA9Xsf2eInHDnVsD3WatserBA1lH2rDhg0ICQnBtm3bUKFCBY3u6+7uXq6TelGcnZ2FLkGrsD3UsT3UsT3UsT1eYluoY3uoK6o9ZDJZkZ1BrxM0lFWpUgUSiaTQpP7k5GRYWlq+9b6bN2/Ghg0b4O/vDxcXF42fWyKRiC6Uie31vgvbQx3bQx3bQx3b4yW2hTq2h7oPaQ9BJ/obGBigfv36apP0lZP2GzZs+Mb7bdy4Eb/88gs2bdoEd3f3siiViIiIqFQJPnw5cuRIzJo1C25ubvDw8MDWrVuRk5ODXr16AQBmzpwJGxsbTJs2DcCLIcs1a9ZgxYoVsLW1RWJiIgDA2NgYJiYmgr0OIiIiog8heCjr3LkzUlJSsGbNGiQmJqJevXrYtGmTavgyPj4eurovO/QCAgKQn5+Pr776Su1xJk6ciEmTJpVp7UREREQlRfBQBgBDhgzBkCFDirxt+/btaj//+++/ZVESERERUZkS/OSxRERERMRQRkRERKQVGMqIiIiItABDGREREZEWYCgjIiIi0gIMZURERERagKGMiIiISAswlBERERFpAYYyIiIiIi3AUEZERESkBRjKiIiIiLQAQxkRERGRFmAoIyIiItICDGVEREREWoChjIiIiEgLMJQRERERaQGGMiIiIiItwFBGREREpAUYyoiIiIi0AEMZERERkRZgKCMiIiLSAgxlRERERFqAoYyIiIhICzCUEREREWkBhjIiIiIiLcBQRkRERKQFGMqIiIiItABDGREREZEWYCgjIiIi0gIMZURERERagKGMiIiISAswlBERERFpAYYyIiIiIi3AUEZERESkBRjKiIiIiLQAQxkRERGRFmAoIyIiItICDGVEREREWoChjIiIiEgLMJQRERERaQGGMiIiIiItwFBGREREpAUYyoiIiIi0AEMZERERkRZgKCMiIiLSAloRynbs2IF27drB3d0dffv2RXh4+Fv3P3z4MDp16gR3d3d069YNJ0+eLKNKiYiIiEqH4KEsJCQEfn5+mDBhAoKCguDi4oLRo0cjOTm5yP3/++8/TJs2DX369EFwcDDat2+PCRMm4O7du2VcOREREVHJETyU+fv7o1+/fujduzfq1KmDhQsXwtDQEIGBgUXuv23bNrRq1QpffPEFateuja+//hqurq74448/yrhyIiIiopKjJ+STS6VS3Lp1C2PHjlVt09XVRfPmzXH16tUi73Pt2jWMGDFCbVvLli1x7NixYj2nQqFQPbdEIil0u0QiQY261aBvUPi2j5WtozVkMhlkMpnG95VIJKjpaA19PcHze4mxs7X8oPaobWtVrtrDwcbivdpDIpGgro0VDCTlpy0AwNHy/dvDycIKBrrlqz1qVHn/9qhZ0Qb6OuXnWGpn9mHHjqpGtpCg/LSHlVHVD2qPioY1oaPQL4XKhGFmaPfG9lBuU2aQNxE0lKWmpkImk8HCwkJtu4WFBR48eFDkfZKSkmBpaVlo/6SkpGI9p1wuBwDcvn37jfu06VYPQL1iPd7H4tq1a+9937YtHAA4lFgt2uBD2sPXwx6AfYnVog3etz261XYAapev9wbw/u3Rw6oGYFWjJEvRCu/bHu0MagMWtUu2GIF9yLGjNtxRW8e95IoRWu6HtYc+OsK85KoRXsa720OZQd5E0FAmBD09Pbi7u0NXVxc6OjpCl0NERETlnEKhgFwuh57e22OXoKGsSpUqkEgkhSb1JycnF+oNU7K0tCzUK/a2/V+nq6sLAwOD9yuYiIiIqJQIOvnBwMAA9evXR1hYmGqbXC5HWFgYGjZsWOR9PD09cf78ebVt586dg6enZ2mWSkRERFSqBJ+ROnLkSOzZswdBQUGIiorCggULkJOTg169egEAZs6ciRUrVqj2HzZsGE6fPo0tW7YgKioKP//8M27evIkhQ4YI9RKIiIiIPpjgc8o6d+6MlJQUrFmzBomJiahXrx42bdqkGo6Mj4+H7iurmby8vLB8+XKsWrUKK1euRI0aNbBu3To4OTkJ9RKIiIiIPpiO4l3rM4mIiIio1Ak+fElEREREDGVEREREWoGhjIiIiEgLMJQRERERaQGGMiIiIiItwFBWhgoKCoQugYiIqFz7mE8qwVBWBpRvkJCQEKSlpQlbjJaQyWTIzc0VugytERsbi/379wtdhuCUn5WUlBRERkYKXI3wlO2RlZUlcCXaJTs7W+gSSMsoL/Sdl5cHHR2djzaYMZSVAR0dHeTn5+P27duoXLmy0OUISvlBuX79Oo4fPy5wNdrj8ePHMDY2FroMweno6AAAwsPDceLECQAvD7ZipGyPkSNHIiUlReBqtMeePXtE/6VOeSw9ffo0Nm7cKHA1wlN+VqZNm4Zz586pfv7Y8OSxZUChUHy0b5DScubMGZiZmaFBgwaibx+5XA5dXV3Rt8OrpFIp5HI5DA0NhS5FKyQlJamuciJ2GRkZ2LVrF8aMGSN0KVohKioKz549g4+PD2QyGSQSidAlCSolJQWVKlX6aNuBoayUKf/gPnjwAPb29tDX1xe6JNIyBQUFSE5Oho2NjdClaA3l54ag+kPLNuEXXHq78vAZ+bir/wgo3yA7duxAUlKSwNUI69X5QhEREQJXoz2uX7+OX375RegyBKd8f0RGRmLXrl0CVyM8ZXscOnSo0DWAxUgZyM6fP4+cnByhy9EKWVlZCAsLE7oMwSmnOFy8eBF3794VuJoPI+5PeRmRy+Xo27cvqlWrJnQpgnp1vlBoaCgAcc8XUnJ1dcXkyZOFLkNwyveHTCZTDdWJ+f2hbI+YmBhYWFgIXI3wlO1x7Ngx0QdUpVu3buHBgwcAPu4Vhx9K+X7477//Pvr3BocvqcyJeb4Qh1+ouPheKaygoADx8fGwt7cXuhStkJWVhdzcXFhYWPD9Uk583JFSSylzrkwmAwDs2rUL8fHxQpZU5t6W9fX09EQZyAAUOmimpqZi7969AlWjfaRSKU6ePCl0GYKTy+XQ0dFBYmIiAHH3grxKT0+PgewVJiYmql5UsQYy5WcjOTkZmZmZAlfz4RjKSoHyw6Fc/RETE4MKFSoIWVKZUX5AXh2KksvlqmEoMc8XCgsLQ2xsrNq28nIgKSkPHjzA/fv3AYg7iCiHYJYvX46IiAjR/sFVfrFVvheWLl2KZ8+eCVmS4JTH0gcPHmDBggXCFiOQoo4NISEhOHv2rADVlCyGshLw6hvk0aNHCA0NxdOnT1XbZs2aBXNzcyFKK3M6Ojr4+++/8ejRIwAvgqmurq7qj4xUKhXdfKGMjAwAwO+//67qIVT+salSpQpGjBghVGmCUygUSEtLU03crlWrFnr06CFsUVpk0qRJcHFxEboMwSi/2Oro6EAul8PW1hbW1tYCV1V2Ll26hNzcXGRmZqqOl8pjqYmJCTp37gxAPMdSJR0dHdy9exe5ubmqLyytWrVCy5YtBa7sw+kJXUB5UFBQAH19faxatQp3796FQqFAZmYmKlasiKpVq6J///5wcnISuswyIZPJsGfPHsTExMDY2Bje3t5o1KgRvL29UblyZbi7u8Pd3R0APvoJmcWRm5uLEydO4OLFi7h37x5iY2Oho6OjCqaLFy/G4sWLYWZmJnClZaugoAA//PADwsLC4Obmhlq1aqFOnTqws7ND3bp1AYh3OOZVdnZ2QpdQppSnNEhJScGff/6J69evY9y4cfDw8ICuri6GDx9eLk57UFyhoaFo0qQJ1qxZAxMTEzg6OsLOzg7VqlWDjY2N6jQ6YmgP5e89KioKx48fx8mTJ1G7dm0sWrQI8fHxqF69OgwMDIQu84MxlJUA5bnHjh8/jlmzZsHW1hbJycmIjo7GlStXRHXNS4lEgs2bNyMiIgKbN2/GyZMnER4eju+//x62trbw8vLClClThC6zzMjlclhbWyM+Ph4ymQzBwcEoKCiAjY0NcnJy8PDhQ1EFMuWBNTQ0FHfu3MG3336LGzdu4Pbt2zhx4gSqVq2KlStXCl2mVhDzxO05c+agffv2uHLlClJTUwG86Gn+5JNP4OjoKHB1ZWfUqFHIz89HTk4O4uPjcePGDRgZGcHCwgK2trYYMGCAaN4jyhGp3bt3w9LSEg4ODqpe01OnTsHExARdu3YVssQSwVD2gRISEvDo0SNYWFjA1dVV1X1as2ZNeHh4oF27dqJazq480eXx48dha2uLKVOmwNzcHP/88w9+//13tUUQH+sZlzVhbGyMZs2aoVKlSkhJSYGenh7u3LmDhIQE6Onp4X//+5/QJZYp5Tf6hIQEdOrUCU2bNkXTpk0BAGlpaaphfzH1hkilUhw7dgydOnVSe81i+WP7Kl1dXeTl5SE5ORn9+vXDvn370KBBAwDAgQMH0K5dO4ErLFvKvx2zZs1Cfn4+oqKiEBERgRs3buDZs2eqazyK4b2i/GyEh4cjICAA06ZNQ6NGjQAAJ0+ehK+vr5DllRiGsg908+ZNbN68GZaWloiOjsbSpUvRs2dP1K1bFwYGBqIKZMDLPyQXL17EtGnTUL16dQBA9+7dcfv2bXh7ewMQR3f7q+rVq6f6v7e3N7KysqCvr18uutvfR+vWrfH777/j7NmzqF27NszNzVG5cmVUrlwZCoVCFO+P3NxcGBoa4saNGwgNDUXnzp1Vf2CzsrKQlpYGW1tbocssc7du3YKLiwtSU1NhaGiIypUrIy4uDgDg4OAgcHVlT/lFVl9fHy4uLnBxcUGPHj0glUoFrqxsKT8X9vb2uHLlCqKiouDj4wMAePLkSbmYTwYwlH0wd3d3TJ06FfHx8bh37x7u3r2LNWvWQF9fH6amphg+fDicnZ2FLrPMKK/h6OXlhbVr12L8+PFwdHSEubk5zpw5gz59+ghdomBenahrYmIicDXCSUtLw+LFi3Hz5k08e/YMtra2cHBwQNWqVdGmTRtR9KACL65ZuH//fpw7dw5OTk5ITEyEmZkZDA0NcfHiRVy9ehVTp04Vuswy5+TkBCMjIwwePBju7u6QSqU4cOCAal6uWHrZlV79kqJQKFQ/K7/QiaGXDHjx2k1MTNCzZ0+sX78eZmZmuHDhAvbv348aNWqUmw4QhrIPZG1tDWtra5w8eRJjxoxBcnIy4uLi8OTJE0RGRoryj6+Ojg5GjBiBn3/+GUFBQSgoKMCtW7fg5eWFOnXqqPYRGzH0/ryNckjy+PHjMDQ0xNGjRxEWFobbt2/j3LlzsLCwENXwlKWlJby9vbFnzx5Ur14dCxYsgEQigb29Pc6ePYvRo0cLXaIgTE1NMXDgQOTm5uL58+fo1KkT2rVrh3HjxgEQ37FDedyQy+WiGaosivJ1u7m5oU2bNjh06BC2bNmCZs2a4dNPPxW4upLDM/p/AKlUCj09PYSFhcHf3x+bNm1S3ZaXl4c7d+7A09NTuAIFlpCQgLCwMMhkMtStWxe1a9eGiYmJKA8seXl5SEpKgo2NDfT0xPldSBnKfvvtN1StWrXQqS+Sk5NhYWEhqvlkcrkc4eHhsLCwQExMDGJjY1XnNRw9ejRMTU2FLrHMFRQUQE9PDxEREUhOToaLiwtMTU1Fc65H4GVv4IkTJ6BQKODj4wMjIyO1fcR4HH3d8+fPUbFiRaHLKFHi/OtQQh48eICzZ8/i9OnTsLa2RlpaGvLy8mBjY4Pz58/j+vXrog5lNjY2RZ5zSmwHkoCAABw+fBgmJiZYvny56o+O2MKZMmglJSXhn3/+QUFBAVxdXVG1alWYm5urLhUjhkCm/KN7+fJlJCcnw9PTU9RnqlcG8djYWOzcuROHDh2Cj48P3Nzc0KBBA1EFMuDl+dnOnTuHgIAA1YhM69at0apVK7i5uYnuOKqkPBm5np5euQtkAE8e+0HMzc1hYGCA6Oho3L17F3PnzsWmTZsQHByM3377rVy+YTShnP8gRsrXffHiRYSGhqJ37954+vQpjI2Nce/ePaxYsULgCoWRlZUFY2NjODs74/Tp09i+fTu2bNmCnTt3ivKbf2BgoOqKDuvWrUODBg2wcuVK5OXlCVxZ2VJ+XpYtWwYDAwOsXr0aTZo0QXBwMObPn4/8/HyBKyx7ubm5yMvLw7Rp0/D999+jS5cu2L17N2bOnImZM2eqTkotNrq6uuX6C235fWVlwNraGkOHDkW1atXg5eWFq1ev4vLlyzh27Bh8fX3Rq1cvoUssM8o/qElJSahYsSIMDAwgl8tFNSH3VcrXfv78eXTo0AGVKlWCq6srgBc9rDExMQJXKAwTExNMnToVBQUFuH//vmp5f2JioqiW9ys/F5GRkfj+++9x7tw5PHr0CBs3bsSWLVsQHx+PGjVqCFtkGXr1ODFo0CDY2NigYcOG6NOnDwYMGIDIyEi4ubkJWGHZefVcfs+ePcOiRYtQUFAALy8vGBsb4+HDh0hLS8OePXtEN+9QDNNAyuerKiPKD0+HDh0AAO3bt0f79u0BiG+FkNK+ffsglUoxceJEUb5+JeVrz8nJQb169fDHH3+oVp5euXIFLVq0ELI8wSh7RPT09ES9vB94MedSX18fe/fuxZEjRzBixAg0bdoUCxcuVJ1KRgyUQTwuLg4KhQLh4eHo2LEjgBd/hDMzM0UVUJVfSrKzs2FgYICMjAyYmpqqtuvo6MDX1xd//vmnkGWWObFMAylfr6aM6erqqoKZcqhOoVBAIpGILpAoDxhNmzbFsmXLcPnyZUyePBkNGzYUuDJhDRo0CNOmTcPNmzfh4+ODR48e4d69exg7dqzQpQmCy/tfkMvlsLGxwZAhQ3D8+HHUrVsXLVu2xJkzZ2BhYSGq89cpf+cPHz5EWFgYTp06BXd3d7i5uSEvLw8DBgwQ1YIHZXu0bdsWJ06cwHfffYdGjRohNjYWly9fxrRp0xAWFqY6cWp5pgzsr04D+f3332FsbIwbN24gJCQEs2bNErrMEsXVlyVILEMv7yKVSvH7778jOTkZvXr1Up2nTSztowzq169fR2pqKhwcHLB3715cvHgRlSpVwqxZs0RzLdQ3eXV5vxgm9r9K+bqV3/SV8vPzsXv3bhgbG4tq6sOrFAoFIiIicObMGZw6dQrR0dHQ1dVFnTp18NVXX4lu4VRqaioCAgJUvYjNmjVDx44dMWnSJHz77bflfnGIcsRpzZo1qF69OqysrHD06FEsWbIE+/fvxz///IN169YJXWaJYij7ANnZ2Xj27JmoutbfJSsrCyYmJnj8+DEmT56MBw8e4NtvvxXVHxllKPv+++9RrVo1DBs2DDk5OWpL2sUSUAEu738TX19f2NraolmzZmjRogXq168PQLxTH7Kzs5GQkABdXV1YWVnB2NgYmZmZuHXrFg4dOoTevXurLrkkBo8fP8bz589Rt25d1fWVlRISElQXIxeD77//Hl27dlVNA2nUqBHmzZsHFxcXDBo0SOjyShSHL9+D8o/u4cOHERYWhuXLlyMiIgJ//fUXmjZtitatWwtdYplStseOHTsQFRWFs2fPws3NDZ6enrCyssLWrVvx6NEjjB8/HoaGhkKXW+qUPT9ZWVmoXLkyAIg2kAFc3l8UuVyOH3/8EeHh4bh06RICAwMBAI0aNcLSpUsFrq7sKI8dDx8+xJYtW3D48GHV2dl79OiBTp06wdvbW3V5tvJOGch/+eUXREREICsrC3l5ebC3t4e9vT169OiB6tWrw8bGRlTHETFNA2Eoew/KzsVTp06hd+/eSE9Px44dO3D37l2Eh4ejSpUqcHd3F7jKsqMMIQ8fPkTDhg0xePBgpKamws3NDYaGhkhPT8eIESPQuXNn0VxyKjMzEzdv3kRgYCAOHTqEVq1awcfHB7Vr1xbNgfRVry7vd3NzQ0REBDZt2oSDBw/Czc0N3377LczMzIQus0wo59F5eHjA3d0dAwcOxNWrV7Ft2zbVFS/EQnks3blzJ3R1dXHp0iU8ffoUwcHB+OOPP1C/fv1yP0T3KuUXmIMHD2L69OmwtbVFUlIS7t+/j8uXL6Ndu3aoXr26KE6w/Po0kGXLlmHv3r3Ytm0bKlWqhIULF5bLa8MylL0H5Qfn8ePHqFmzJgIDA2Fra4vFixdj/PjxyM7OFrhCYXzzzTfIz88v1NVeqVIlVK9eHTVr1hSosrKnr6+PFStW4PHjx7h27RouXLiAP/74A/Xq1cNPP/0kdHllhsv71Sl7NxITE2FlZQUdHR1IJBI0btwYx44dE10oUwYLIyMj+Pr6AgCqVq2KcePGITw8HOfPn4e9vb0oQojyvREbGwtPT0/VSn4A8PLywmeffaYasizvbfGqI0eOoFq1amjbti0mTZpU7kcdGMrek1QqRdu2bTFjxgykpqZi165dyM7ORlxcHBo3bix0eWVG2d1+9epVXLhwAXFxcTA1NYWzs7PqX0FBAebOnSuaFWUKhQIVKlSAvb09atasiVatWkEmk+Hhw4eiOwkml/er09HRQV5eHkaOHIlKlSqhfv36aNasGerXr48TJ06gf//+QpdYpnR0dJCTk4MTJ06ormwgkUiQnZ0NmUwmmmFL4GXAiIyMxJUrV7BhwwZ07twZdnZ2MDY2hrGxsdAllimxTgPhRH8NKSeyK129ehUVK1ZE7dq1cezYMQQFBZW71SBvo/wGqzyJrrOzMzIyMpCUlIS4uDiMGTMGzZo1E7rMMnf58mVcvHgRqampsLW1RZMmTVCjRg1RXqAeAFJSUjB//nyYmJgUuby/UqVKGDp0qNBllgmpVIqIiAhERUXh2rVruHXrFp4+fYo+ffrg66+/Frq8MpeYmIjFixfjxo0bSE5Ohr29PYyMjODp6Ym5c+cKXV6ZCw0NxaFDh/Ds2TMYGhrCzMwMVlZWGDRokKiGcoEX00CGDRuGyMhItGzZUm0aSHnFUKahMWPGYOnSpfjrr79gb2+PVq1aqXqAkpOTkZmZCUdHR4GrLFs5OTkYM2YMtm/fDuDFH+CEhAQ8fPgQrVq1gqmpabn9VvMq5Wu8e/cuFi5cCBcXF5w5cwYeHh4IDw9HjRo18Ntvv5X7dngTsS/vf9XrnwepVKrqYRWziIgInD59GleuXMH9+/eRkJCAMWPGYNKkSUKXVmaUow8pKSmIiIjAvXv3cOfOHUyaNKlczqF6m7y8PMTFxammgdy9exf37t0r19NAGMo0lJaWhsqVK2PevHk4f/480tLSYGtri7Zt26Jt27aiWrKt/MNy584drFq1CtOnT0fdunWFLkswyoPphg0bYGRkhDp16uDPP//EjBkzsGTJErRu3Vp0w1NKXN7/0qlTp3Ds2DGkpqbC09NTNXwpZrdu3YKNjQ0sLS1V27Kzs3Ht2jWYmprCw8NDwOrKjlwuR1RUFIKDg1GjRg20b98e5ubmovuMAEWfz+/VaSAuLi4CV1g6GMo+gEKhQExMDC5evIh///0XDx8+xOHDh4Uuq8woPzQhISH49ddfYWxsjGbNmsHOzg6Ojo5wc3MT1TwIZXuMGTMGEydOxKFDh1CvXj306NEDP/30EywtLUUzRAcUf3k/UH7nh7wuPj4e/fr1w5gxY1BQUIDr168jJiYGOTk52LdvX6Hzt5V3aWlp2LJlCx4+fIj79+/D3NwcTZs2hYeHB9q2bSt0eWVG+VnZtm0bwsLCULlyZdy7dw9PnjzBzJkz0bNnT9F8Rl4lxmkgnOj/AXR0dODo6AhHR0f07dsXBQUFQpdUppQHCHd3d3z77beIjo7Go0ePcO3aNRw9ehSTJ08WVQ+Asj169OgBe3t7WFlZ4d9//4WpqSn+/fdffP/99wJXWLa4vP8l5R/U69evo2PHjhg6dCiys7PRq1cvpKSkIDk5WVSBTPk7P3r0KJKSkuDl5YWcnBw0bdpUFdLatm0rmiCifP//888/mDhxomoe7pkzZ7B79240atQIDg4OQpZYZl6dBvLTTz+pTQPZtWtXuZ8GwlBWgsrbhVGLS9nz0bRpUxQUFCA+Ph73798X7VBm586dAQD9+/dHXFwcDh48CG9vb7i6ugpcWdnh8n51yj8gjo6OiI6OBgBVL3KlSpVEdboY4OX5yc6fP4+RI0fi9OnT8Pb2xhdffIHExES0aNFCtV95/eP7Kh0dHWRlZSErK0ttYVTLli2xcuVKUc01lMvlkEgkCA0NRadOnVCnTh2kpqZi6tSpqmkg5fk9Ic4UQR9M+U33yZMnOHbsGGJjY2FhYYE6deqgTp068Pb2Fs0pMN7EzMwM48ePR35+vmqYTiy4vL9oN2/exK5du7Br1y74+PigVatWaNasmdpcKjFQ9qJmZGTAysoK9+/fR9++fQEA9+/fxyeffAJAPBeol8lk0NPTQ6NGjTBkyBCMHTsWjo6OSE5ORkFBgajmkym/pF2+fFk1DaR169aoVq0a6tSpA6lUKnCFpYtzyjR04sQJREdHY9SoUUKXIijlHIjZs2ejatWqOHToEKpWrYqUlBTo6elh6tSpaNOmjdBlksC4vP9lQL19+zbGjBmDVatWITMzE5cvX8Z///2HhIQEHD16VBQ9hq978uQJzM3NceTIEaxduxb169dHdHQ0goODRXn9z+TkZGzZsgUpKSm4c+cOzMzMMHr0aLRt21Z010QNCQmBj48PAgMDER4eju7du2P16tX4/vvvy/WoA3vKiunRo0ewtrbGiRMnVCuBlKtCLl68CFNT03L9Rnmd8uAQGRmJZcuWITw8HAsWLMCzZ8+watUq2NnZARDP8AMVrVWrVmjbtm2h5f1iDCCJiYkYNWoUGjdujPz8fLRu3Ro5OTnIzMwUZXsAUJ3ioWfPnjA0NMSTJ08wcOBASCQS0Rw7Zs+eDXd3d/j4+KBWrVqYMWMGoqKiYGRkBGNjY9WJU8UUyADxTgNhKCumq1evYuHChcjNzcWTJ09gbGyMJk2awMrKCmvWrMHAgQPL/Zvldffv30elSpWQmZmJ5ORkODg4wMHBARUqVECNGjUAiGf4AWAv6uvkcjkePHigtry/efPmolver5wjc/ToUTx//hxPnz5F1apVAQAmJibleiVZUZRh6+HDhwgKCoK+vj4+/fRTfPbZZ4X2Ke8yMzNhZWWFCxcuIDAwEAYGBnBzc0OLFi3g7u6uCmRiJrZpIBy+1EB6ejqmTZuGGjVq4N9//0VcXBwcHR1RvXp1rF27VnQH16SkJERHR6NmzZpYtmwZPDw8kJ6errrOoxhW1QEve1H9/Pzg4eGBPn36iLoXlcv7i+bn54d///0Xz549Q61atdCqVSu0bt1aVJdlUyooKEDPnj3h7e2N+/fv49q1a5BIJHBwcMCWLVtQpUoVoUssM1KpFFlZWUhMTMT9+/cRHh6Oe/fuITY2Fk2bNsWSJUuELpHKEENZMSkvtJ2Xl6daCZOXl4eLFy/C2toazs7OAlcorNDQUGzYsAFOTk5o37696nqPYuhyDw4OVvWiNm/eHL1791b1og4ZMgQDBw5Ely5dhC6zzCgD15AhQ4pc3j9jxgzRLO8vyqNHj3Dx4kWcOHECDx8+REhIiNAllblbt27hxx9/xO+//67advv2bZw+fRpjx44VrjCBKRQKSKVSPH/+HDExMTAwMIC7u7tojqXEUFYsyj8yeXl5CAkJQXZ2Nry9vVGnTh2hSxNURkYGzpw5AxMTEzRu3BgGBgaiPS0Ie1HVZWVlYciQIQgKClLb3qtXL/z666+iGr58m1fPVi4GW7duhbGxMUxMTJCYmIjhw4cLXZJWEMuoQnGIfRqIeI4GH0A5zDJp0iQ4OTlh+/btMDIygpmZGezt7fHjjz/CwsJC4CrLhvLgcf36dSxduhS2trbIz8/H1q1bYWlpCTc3N1GdtR540YtaqVIlrFu3DhUqVMDcuXPVelHFFsi4vL/4xBTIACA3Nxd79+5Famoq0tLScPnyZfTp0wfNmjUT1bm4XsdAxsV0SuI6IrwHZS/ZzZs3IZfL0b9/f/z3339Ys2YNZs+ejQoVKogmkAEvT/p47NgxNG3aFCNGjEB8fDyePn2qtqpOLN/8FAqFalj79V7UVq1aCV2eICQSCSQSCcaPH48tW7YgJCREtbx/6tSpAMDhGJEaO3Ysxo4di8TERFy7dg3//vsv/Pz8kJqaisqVKyMwMBCmpqZCl1kmlMfIu3fvwsTEBLa2tpDL5QDEGdK4mO4FDl++g/KDo7zItKOjI/bv348VK1bg8OHDiImJEeUciNWrVxe6AHtGRgZ0dXVF1zMEAGPGjBF9LypQeHk/gCKX94uJ2IdjiuPhw4e4efMmunbtKnQpZW7Dhg1ITEzEiBEjVKcIEStOA2FP2Tspv7EYGhrCzc0NcXFxMDMzA/DiEiFWVlZClicIhUKBNm3aYO7cuejevTuaN2+O+vXrq9pFLNiLqo7L+9VxOKb4atSooTqNjth06tQJgYGBGDx4MBo1aoThw4fDzc1NdL1lnAbyAnvKNCCVSpGamorJkyfj/v37cHV1xXfffSeaM5Mrew2VZ992c3ODjo4Onj59itzcXDRq1AjTp08Xuswyw17Uwri8/yWuyqU3eXXxWFZWFipWrIiMjAysXr0ad+/exYABA9C9e3ehyywzXEz3EnvKNGBgYAAbGxsEBATg3r17qFy5sih7yq5fv46BAwdi8ODBiIuLQ2pqKu7fv6/6JiOW+ULsRS3MwMAABgYGqFKlCurWrYv27durLe8HxPP+6NGjBz755BPVcMzy5cvVhmPatm0rdIllRvlHNz09HfHx8XBxcRG6JEEpTyj866+/4u+//4aenh7MzMzg4OCAx48fIzQ0FN27dxfNZ4WL6V5iKHsL5QfiwoUL2LJlC6KiotCiRQu0bdsWHh4esLCwENVJMJUhRCqVqs7LVr16dVSvXh0uLi6qRQBiOIi8atiwYZBKpahevTq2b9+Oxo0bw9XVFV988YXQpQlC2YOoo6ODChUqwMrKSi2giuX9weGYl169qsGVK1fg5+cHqVQKAwMD5OXlQU9PTzTvC+DlZ8DW1hZVq1ZF586d0bRpUxgaGmLRokWqvyliGMLkNBB1HL58C+X8j88++wxjx46FkZERTp06hWvXriEqKgrr168X3UW309LSMGDAACQkJKBTp05o2bIlmjZtKspeoTcRcy8qvcDhGHVHjhzB4cOH8fDhQ/Tt2xdDhgxR3bZmzRpUrlwZw4YNE7BC4cTFxWHLli149uwZOnbsiObNm4sqhHAaiDr2lL2F8hxCjo6O6NGjBwDA19cXAJCQkCC6icvAi6G6lStXIjo6Gjdv3kRQUBBWr14tuvlC7EVVx+X96jgco87b2xsxMTE4d+4c/Pz88PPPP6Nx48b47LPPcObMGUyZMgWAOE6lo3yNN2/exN9//42GDRvC29sbgYGBmD17NgYOHIi5c+cKXWaZ4TQQdQxlb7Bjxw6sX78enTp1gqWlJS5cuABvb2/V7WI9AaahoSFcXV3h6uqKDh06iHa+kLKDecGCBWq9qMuXLxdlL6rywBoaGir65f0cjimsSpUqGDNmDGxtbdGlSxeEhYXhyJEj2Lx5Mz799FP4+PgAEEeAV77GpKQkPHnyBLdu3YKtrS2aNm2KSpUqwdraGoD4rvbAaSAvcPjyDTIzMxEUFITQ0FBcv34dOTk5aNasmeq6jmJZcfk6MXyT1cS4cePw22+/qW1T9qKK8QzlMTExCAwMxP79+0W7vJ/DMYUpv6xlZWUVmksnph7lt1FeX5leEOs0EIayN/jzzz/Rt29f1c+3b9/GkSNHcOrUKURERMDf31/17Y7E5dVe1OzsbHTr1k2tF1VsuLy/aNu2bYO7uzvi4uJw6dIlLFiwAPPnz4eVlRUmTpwodHllRhlSDx06hPPnz+P8+fOoWrUq2rZti5YtW8LZ2Vl0X/akUilOnTqFo0ePwszMDI6OjujVq5eoFn8AnAZSFIayIiQkJGDmzJkwNDREjx490LFjx0LdyGJ6o3C+kDr2oqpTHlhXrVpVaHn/uXPn0LhxY6xcuVI0Q9uvEvu5DZWkUil69uyJ+fPnY968eejUqRMCAgKQlpaGv//+G46OjkKXWCaUn4E9e/bgxIkTqFGjBqpXr47Tp0+jUqVKWLRoEYyMjIQus8xwMV1hDGVvEB0djT///BMnTpyAm5sbunXrhpYtW4oyhCjxciAvsBe1aH/++SdCQkLUlvdXqVIFOjo60NfXF9UXmaKIcThG+YXuzJkzOHjwIGbMmIGJEyciICAAAQEByM/Px9ChQ4Uus8wo22P8+PEYOXIkmjZtCuDFqvZvv/0WgwYNEuWxg9NAXhJvwniHmjVrYubMmdi+fTuqV6+OHTt24LfffsOTJ0+ELk0wnTp1grGxMQYPHoxp06YhPDxc1WMmFgkJCfjrr78wduxYHD58GAUFBXB1dcXUqVMRHByMiIgINGvWTOgyBdG3b1989913iIyMxIoVK3Dx4kVkZGSo5smIIZDJZDIAwIULFzB27Fh06NAB8+fPx4kTJ2Bubg4rKyuI6Xuw8nceGRkJb29v/Pfff3BycgLwYvJ/eHg4AIjmOKL8Ul9QUIDExEQAL0ZdKleujNTUVNWCKTG8R3bs2IHWrVtj6dKlqsV0r7KxsRFdIAPYU1aITCbDtWvXkJCQgJycHLi5uSE3NxeBgYHYs2cPzMzMEBwcLJqeIs4XKoy9qC8VtbxfJpMhMDAQp0+fFt3yfg7HFO3u3bswNDREbm4uZs+eDWdnZzx+/Bg9evRA7969RTe0ferUKWzatAmff/456tevj/PnzyMkJAR79uwRurQyw2kgRWMoe83Zs2cxevRo1akw7ty5g1q1aqFu3bqIiorCrVu3sHfvXqHLLDOcL/RmSUlJ2L59OyIiItCgQQN8/vnnognrrwsNDcWBAweQlpYGW1tb1KxZE5GRkahduzbGjBkjuuX9HI55s5MnT+Lw4cNwcXFB//79RTOH6vnz59DT04OxsTHy8/Oxd+9e7NmzBxkZGWjbti06duwIb29v0Sx64DSQojGUvSYzMxN79+7FnTt30KZNG/j6+kImkyEvL091Qjsx4nwh9qJqQozL+7kqV53ymJCdnY2wsDDk5+ejfv36ou0BWbp0KczMzODq6opatWqhZs2aAF70NstkMtEcRwEupnsbhrIi5Obm4sKFCwgMDERGRgY6dOiAzz77DObm5kKXJiixXw6EvahF4/L+Fzgco04ZzL///ntEREQgJiYGOTk5qFq1KmrXro3Ro0eL5sLkMplM1av++PFjpKenw83NDXXr1kW9evVgbW2NmjVriqKHTInTQIrGUPYKmUwGqVSq1p1+9epVXL9+Henp6fDx8VGtlhEDzhdSx15UdVzer47DMUXz9fXFjh07YGlpiby8PJw9exZBQUEYNmwYmjRpIprhOqVTp05h5cqV6NWrFy5duoSEhAQ4ODhg+fLlQpcmCE4DUcdQ9oorV65g5cqVSE9Ph62tLezt7fHs2TNcvXoViYmJsLa2xqlTp4Qus8xxvtBL7EV9icv7X+JwjLrHjx/jwYMHcHNzw86dOzFkyBBRXiv4VVKpFAYGBvjpp59QpUoVjBgxAsCLUPL06VO4ubmJIqByGsjbMZS9IiIiAhERETA1NUVUVBQAoGrVqqhQoQJiY2NhZ2eHzz77TOAqhSfG+ULsRX2zL7/8Ej169ECXLl1UwWPIkCGYMmUKGjVqJJowwuGYl44dO4aJEyfCwMAAEokE3t7emDp1qup0GGIjlUqhr68PHR0dDBs2DDNmzIC7u7soF0hxGsjbMZTRW3G+0AvsRX0zLu9Xx+EYqK5xmZCQgBMnTuDw4cO4evUqKlSoADs7OyxcuBAeHh5Cl1kmYmNjMX36dLRr1w516tTB6tWrceDAAaHLEgyngbwdQ1kxiaFb+VWcL6SOvajquLz/JQ7HqMvPz8f06dPh6+uLzp07q90WERGBwMBA9O3bF05OTqJ4f2RkZGD37t24fv067t69i7S0NLRt2xb169dHw4YNUbNmTZiamgpdZpniNJA3YyijInG+EL0Nl/e/xOEYdZmZmfD390dQUBB0dXUxbNgwdOjQAdWrVxe6NK1w584dHDt2DP/99x+uXbuGiRMnYvTo0UKXVSY4DeTdxDE7mzSmyeVAxPCH923E8G3/VTKZDNWrV0dERAQuXLgg+uX9DRo0wOzZs3Hnzh14eXlhzpw5oh6OMTU1xaRJkzBp0iQEBATg0qVLSElJQe/evUV3WpCi1KtXD/Xq1QPwYnpIdnY2AHEcR65du/bWaSCBgYGinQaixJ4yeivOF6K34fL+Fzgc85JUKkVGRobqHIZnz57F5s2bcevWLbRr1w5ff/01bGxsBK6ShMBpIO/GUEaFcL4QvQuX97/A4Rh1ycnJmDp1Kuzs7BATEwM9PT3VCtTdu3fj4cOHOHbsGOzs7IQulUgrMZRRIZwvRG/D5f0vcVWuunv37mHKlClwd3fHiBEjoFAo8OjRIxgbG8PGxgaJiYlo0aKF0GWSFhLDl7jiYCgjNbwcCL0Nl/er43BMYffv31fNJWvbti26d++O2rVrq27nFzqiN2MoozfifCF6HZf3U3ElJCTg+PHjyMjIgIeHB1drExUDQxkVwvlCVFxiXt7/LmL7jKSmpuLAgQOwtLSEvb09srOzERUVhWvXruG///6DhYUFtm3bBkNDQ6FLJdJaPCUGqVHOFwJeTFieMWMGgBfDmpaWlrC0tAQAUf2xoTcT8/L+dxHb679z5w7CwsKQlJSExMRE1K9fH5UrV0ZcXBwyMzNRs2ZNGBoacviS6C0Yykjl9flCaWlpcHd3BwDRTeAmzRkYGKjOXye2QEKAt7c3GjRooLoEW3x8PBQKheqksampqQA4p4zobTh8SSqcL0REJY0hjKj4GMrojThfiIiIqOwwlFGxKOcLVa5cmfOFiIiISgFDGREREZEWYHcHERERkRZgKCMiIiLSAgxlRERERFqAoYyIiIhICzCUEREREWkBhjIi0npDhw7Fd999V2j7vn370LhxY62pp7T9/PPP+Pzzz8v8eYmobDCUERG9QX5+vtAlEJGIMJQRUblx4cIF9OnTB56enmjcuDEGDBiAJ0+eqG4/duwYevbsCXd3d7Rv3x5r165FQUGB6nZnZ2fs3LkT48aNg6enJ3777bdiPW+7du3w22+/Yc6cOWjYsCHatm2L3bt3q25//PgxnJ2dcejQIQwYMADu7u7o2rUrLl68qNqnqF6/Y8eOwdnZWXX72rVrERERAWdnZzg7O2Pfvn3v1U5EpJ0YyoioXCgoKMCECRPQpEkTHDhwALt370b//v1V1128fPkyZs2ahWHDhiEkJASLFi3Cvn37CgWvtWvXomPHjjh48CB69+5d7Of39/eHm5sbgoODMWjQICxYsAAPHjxQ2+eHH37AyJEjERwcDE9PT4wbN051oe536dy5M0aNGoW6devizJkzOHPmDDp37lzs+ohI+zGUEVG5kJmZiYyMDHzyySdwcHBA7dq10bNnT1SvXh3Ai7A1ZswY9OzZE/b29mjRogUmT56MgIAAtcfp2rUrevfuDXt7e9V9i6N169YYPHgwHB0d8eWXX6JKlSq4cOGC2j6DBw+Gr68vateujQULFsDMzAx79+4t1uMbGhrC2NgYEokEVlZWsLKygqGhYbHrIyLtpyd0AUREJaFy5cro1asXRo8ejRYtWsDHxwefffYZrK2tAQARERH477//1HrGZDIZ8vLykJOTAyMjIwCAm5vbez2/cpgRAHR0dGBpaYnk5GS1fRo2bKj6v56eHtzc3Ar1phGReDGUEZHWMzExQWZmZqHtz58/h5mZmepnPz8/DB06FKdPn8bhw4exatUq+Pv7w9PTE9nZ2Zg0aRI+/fTTQo9ToUIF1f+NjY3fq0Y9PfXDqY6ODjS5tLCurm6h/bnQgEhcOHxJRFqvZs2auHXrVqHtt2/fRo0aNdS2ubq6YuzYsQgICICTkxP++usv1fbo6Gg4OjoW+qerWzaHwmvXrqn+X1BQgFu3bqFWrVoAgCpVqiArKwvZ2dmqfSIiItTur6+vD7lcXia1ElHZY08ZEWm9QYMGYceOHViyZAn69OkDAwMDnDx5EocOHcKvv/4KAIiNjcWePXvQrl07WFtbIzo6Gg8fPlSd12vChAkYN24cqlevDl9fX+jq6iIiIgJ3797FlClTyuR17Ny5EzVq1ECtWrWwdetWpKenqxYTNGjQAEZGRli5ciWGDRuG69evF1pdaWtri8ePH+POnTuwsbGBqakpDAwMyqR2Iip9DGVEpPXs7e3xxx9/YNWqVRg5ciTy8/NRq1YtrF69Gq1btwYAGBkZ4cGDBwgKCkJaWhqsra0xePBgDBgwAADQqlUr/Pbbb1i3bh02btwIPT091KpVC3379i2z1zFt2jRs2LABd+7cgaOjI3799VeYm5sDeDEn7scff8QPP/yAP//8Ez4+Ppg0aRK+/fZb1f19fX1x9OhRDBs2DM+fP4efnx969epVZvUTUenSUWgy6YGIiDT2+PFjtG/fHsHBwahXr57Q5RCRluKcMiIiIiItwFBGREREpAU4fElERESkBdhTRkRERKQFGMqIiIiItABDGREREZEWYCgjIiIi0gIMZURERERagKGMiIiISAswlBERERFpAYYyIiIiIi3AUEZERESkBf4f+fWrwvVkZBsAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_metric('context_precision')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "8c96efbc-2d1c-424b-a045-4dff88985669", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAHMCAYAAACUdN+cAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABli0lEQVR4nO3dd1QUZ/828AsWkKpIVRGwAtIUG2Kvj0ajscaGPbE86mOMiZo3JtGoQRM11l+isZdYIkI0liTYC0o0KDZAUAQFkar0hd19//Dsygoqq8CMzPU5x3PCbJnv3pndvfYuM3oqlUoFIiIiIhKUvtAFEBERERFDGREREZEoMJQRERERiQBDGREREZEIMJQRERERiQBDGREREZEIMJQRERERiQBDGREREZEIMJQRERERiQBDGRG9c0aNGoVRo0Zp/n7w4AFcXV1x4MCB1z42Li4O48ePR4sWLeDq6oqQkJCKLFUQc+fOhY+Pj9BlEJGOGMpI8uLj4/H111+jW7du8PLyQvPmzTFs2DBs27YN+fn5FbbfmJgYrFmzBg8ePKiwfaj9/PPPZQ4f6oCj/ufm5obWrVvjo48+Qnh4eAVXWvHmzp2L6OhozJw5E99//z08PT0rbF8vtuWL/zZs2FBh+34bXbt2xaRJk0q97fr162UOwJXhwIEDcHV1xfXr14UuBQCQl5eHNWvW4NKlS0KXQu8gA6ELIBLSqVOnMGPGDBgZGeGDDz6Ai4sLCgsLceXKFfzwww+IiYnBwoULK2TfMTExWLt2LVq3bo26detWyD7U1q9fj549e6J79+5lfsz777+Pjh07QqlUIi4uDr/++itGjx6N/fv3w9XVtQKrrTj5+fkIDw/H5MmT4e/vX2n7Vbfli9zd3SutBqoceXl5WLt2LaZNmwZfX1+hy6F3DEMZSVZCQgJmzpyJOnXqYNu2bbCzs9PcNnLkSNy/fx+nTp0SrkCBubu744MPPtD83aJFC3z88cfYvXs35s+fL1xhbyE9PR0AUL169XJ7ztzcXJiamr7yPi+2JemmLG1MVBVw+JIka+PGjcjNzcXixYu1Apmas7MzxowZo/m7qKgI69atQ/fu3eHp6YmuXbtixYoVkMvlWo9TD/1cvnwZgwcPhpeXF7p164bg4GDNfQ4cOIAZM2YAAEaPHq0Zzio+5HH69GmMGDECzZo1g4+PDyZOnIg7d+5obg8NDYWbmxtWrVqltf9Dhw7B1dUVv/76KwDA1dUVubm5CAoK0uxn7ty5OrdXy5YtATwLs8U9ffoUixcvRqdOneDp6YkePXpgw4YNUCqVWvdTKpXYtm0b+vbtCy8vL7Rp0wYTJkzQGnYKDAzE6NGj4efnB09PT/Tu3VvzOt7WmjVr0KVLFwDA999/D1dXV3Tt2lVz+61bt/DRRx+hefPm8PHxwZgxY3D16lWt51APlYWFhWH+/Pnw8/NDp06dyqW+kJAQTJw4Ee3bt4enpye6d++OdevWQaFQlLjvtWvX8PHHH6NVq1Zo1qwZ+vbti23btpW4X3JyMv773//Cx8cHbdq0wdKlS0t9vreVkpKCL774Ah07doSnpyfat2+PKVOmlBiaf90xDTyfDxcfH4+PP/4YPj4++Oyzz3SqR/0cr3v96uHlTZs2YevWrejSpQu8vb3h7++P6Ohored8cR5j8X2pj6MHDx7Az88PALB27VrN+23NmjU61U/SxZ4ykqyTJ0/C0dERzZs3L9P9582bh6CgIPTs2RPjxo1DREQE1q9fj9jYWKxbt07rvvfv38eMGTMwePBgDBgwAIGBgZg7dy48PDzQuHFjtGrVCqNGjcKOHTswefJkNGjQAADQsGFDAEBwcDDmzp2L9u3b47PPPkNeXh52796NESNGICgoCHXr1oWfnx9GjBiBDRs2oHv37vDw8MDjx4+xaNEitG3bFsOHDwfwLIDMmzcP3t7e+PDDDwEATk5OOrfXw4cPAWj3MuXl5cHf3x/JyckYNmwYateujfDwcKxYsQIpKSn48ssvNff98ssvceDAAXTs2BGDBw+GQqHA5cuXce3aNXh5eQEAdu/ejcaNG6Nr164wMDDAyZMnsWDBAqhUKowcOVLnmovr0aMHLCwsEBAQoBlONDMzAwDcuXMHI0eOhJmZGT766CMYGBhg7969GDVqFHbu3ImmTZtqPdeCBQtgZWWFqVOnIjc397X7zsvL0/TSFVe9enUYGDz7GA4KCoKpqSnGjRsHU1NTXLx4EatXr0Z2djbmzJmjecz58+cxadIk2NnZYfTo0bCxsUFsbCxOnTql9SNCoVBgwoQJ8Pb2xuzZsxEaGorNmzfD0dERI0aMeKM2fJnp06cjJiYG/v7+cHBwQHp6Os6fP4+kpCTN0HxZjmm1oqIiTJgwAS1atMCcOXNgbGysc026vP7g4GDk5ORgxIgRKCgowI4dOzBmzBgcOnQINjY2Zd6nlZUV5s+fj/nz56NHjx7o0aMHALyzw/0kABWRBGVlZalcXFxUU6ZMKdP9b9++rXJxcVF9+eWXWtuXLFmicnFxUYWGhmq2denSReXi4qL6559/NNvS0tJUnp6eqiVLlmi2HT16VOXi4qK6ePGi1nNmZ2erWrZsqZo3b57W9pSUFFWLFi20tufm5qp69Oih6tOnj6qgoEA1ceJEVfPmzVUPHz7UemyzZs1Uc+bMKdNrTUhIULm4uKjWrFmjSktLU6WkpKj++ecf1aBBg1QuLi6qo0ePau67bt06VbNmzVT37t3Teo5ly5apmjRpokpMTFSpVCpVaGioysXFRbVw4cIS+1MqlZr/zsvLK3H7+PHjVd26ddPa5u/vr/L39y9Rc2BgYJle28aNG7W2//e//1V5eHio4uPjNduSk5NVPj4+qpEjR2q2BQYGqlxcXFTDhw9XFRUVvXJfxff3sn/h4eGvfO1fffWVqmnTpqqCggKVSqVSFRUVqbp27arq0qWL6smTJ1r3Ld6Oc+bMUbm4uKjWrl2rdZ/+/furBgwY8Nq6u3Tpopo4cWKpt0VERGi19ZMnT0pt0+J0OabVtS9btuy1dapUz/+fRERElHiO171+9f8fb29v1aNHjzTbr127pnJxcVF99913mm0vHnPF99WlSxfN32lpaSoXFxfV6tWry1Q/UXEcviRJys7OBgBNT8nrnD59GgAwbtw4re3jx4/Xul2tUaNGmuE+4Nkv6Pr165cY+ivNhQsX8PTpU/Tp0wfp6emaf/r6+mjatKnWEKeJiQkCAgIQGxuLkSNH4tSpU/jiiy9Qp06dMr2uV1mzZg38/PzQrl07jBw5ErGxsZg7dy569eqluc+xY8fQokULVK9eXavWtm3bQqFQ4J9//gEA/PXXX9DT08O0adNK7EdPT0/z38V7RLKyspCeno7WrVsjISEBWVlZb/2aSqNQKHD+/Hl0794djo6Omu12dnZ4//33ceXKFc3xovbhhx9CJpOVeR9Dhw7Fli1bSvxr1KiR5j7FX3t2djbS09PRsmVL5OXl4e7duwCeDbE+ePAAo0ePLjEvrng7qql7S9VatGhR7qt9jY2NYWhoiLCwMDx58qTU++hyTL+s9jdR1tffvXt32Nvba/729vZG06ZNS7yviSoahy9JkszNzQEAOTk5Zbr/w4cPoa+vX2LYz9bWFtWrV9cM7anVrl27xHPUqFHjpV9axcXFxQGA1lBUcera1Vq0aIHhw4dj165daN++PQYPHvzafZTF0KFD0atXLxQUFODixYvYsWNHiflI9+/fR1RUlGYezYvUQ3bx8fGws7ODpaXlK/d55coVrFmzBlevXkVeXp7WbVlZWbCwsHjzF/QS6enpyMvLQ/369Uvc1rBhQyiVSiQlJaFx48aa7bqulnV2dkbbtm1feZ87d+5g5cqVuHjxYokQqA6k6lDv4uLy2n1Wq1YNVlZWWtvKegyWhToEGhkZ4bPPPsPSpUvRrl07NG3aFJ07d0b//v1ha2sLQPdj2sDAALVq1Xqr+nR5/c7OziW21atXD0ePHn2rGoh0xVBGkmRubg47O7sSk4xfp7TeiNLo0ovyIpVKBeDZXDD1l9qrnlsulyMsLAzAsy/tvLw8mJiYvPH+1YoHiS5dukBfXx/Lly+Hr6+vZg6YUqlEu3bt8NFHH5X6HPXq1Svz/uLj4zF27Fg0aNAAc+fORe3atWFoaIjTp09j69atJRYOCKlatWrl+nxPnz6Fv78/zM3N8b///Q9OTk6oVq0abt68iWXLlr3Ra3+bY9DIyOil5+hTby/eBmPHjkXXrl0REhKCc+fOYdWqVdiwYQO2bdsGd3d3nY9pIyMj6Ou/3UDO27x+XVTEwgmSLoYykqwuXbpg7969CA8Pf+3Zzx0cHKBUKnH//n3NZHwASE1NxdOnT+Hg4KDz/l8W8NRDaNbW1q/tXQGA1atXIzY2FnPmzMGyZcuwfPlyzJs3T+d6XmfKlCn47bffsHLlSmzatAnAswUDubm5r63TyckJ586dQ2Zm5kt7y06cOAG5XI6ffvpJa/i1ok/CaWVlBRMTE9y7d6/EbXfv3oW+vn6pPZ/lKSwsDJmZmVi7di1atWql2f7iUJv62IiOji7TsfGmHBwcEBMTU+pt6nZ6cYjcyckJ48ePx/jx4xEXF4f+/ftj8+bNWLZsmc7HdGW7f/9+iW1xcXFa7+saNWqUOv0gMTFR6++y/nAjKg3nlJFkffTRRzA1NcW8efOQmppa4vb4+HjNaQbUpz148bQDW7Zs0bpdF+rerBfnSnXo0AHm5uZYv349CgsLSzyu+Cq+a9euYfPmzRgzZgzGjx+PCRMmYOfOnZqeMzVTU1M8ffpU5xqLq169OoYOHYpz587h9u3bAID33nsP4eHhOHv2bIn7P336FEVFRQCA//znP1CpVFi7dm2J+6l7UdQ9G+q/gWdtExgY+FZ1v45MJkO7du1w/PhxrRCUmpqKP/74Ay1atCgxvFbe1L1CxV+7XC4vcToQDw8P1K1bF9u3by/x/7P4Y99Wx44d8ejRoxJXgZDL5fjtt99gbW2tOfFtXl4eCgoKtO7n5OQEMzMzzelidDmmhRASEoLk5GTN3xEREbh27ZrWCX8dHR1x9+5drVojIyPx77//aj2X+n39tu83kib2lJFkOTk5YdmyZZg5cyZ69+6tOaO/XC5HeHg4jh07hoEDBwIA3NzcMGDAAOzduxdPnz5Fq1atcP36dQQFBaF79+5o06aNzvtv0qQJZDIZfvnlF2RlZcHIyAht2rSBtbU15s+fj9mzZ2PgwIHo3bs3rKyskJiYiNOnT6N58+b4+uuvUVBQgDlz5sDZ2RkzZ84E8OzUBCdPnsQXX3yBQ4cOaU646eHhgdDQUGzZsgV2dnaoW7duidM8lMXo0aOxbds2bNiwAT/++CMmTJiAEydOYPLkyRgwYAA8PDyQl5eH6Oho/Pnnnzh+/DisrKzQpk0bfPDBB9ixYwfu37+PDh06QKlU4sqVK/D19YW/vz/atWsHQ0NDTJ48GcOGDUNOTo4mAKSkpOhcqy4++eQTXLhwASNGjMCIESMgk8mwd+9eyOVyfP7552/9/Ldu3cLvv/9eYruTkxN8fHzg4+ODGjVqYO7cuRg1ahT09PTw+++/lwha+vr6mD9/PqZMmYL+/ftj4MCBsLW1xd27dxETE6PpwXxbQ4cORWBgIGbMmIFBgwahSZMmyMzMxJEjR3Dnzh0sXboURkZGAJ71KI0dOxa9evVCo0aNIJPJEBISgtTUVPTp0wfAs+kCZTmmheLk5IThw4dj+PDhkMvl2L59OywtLbWG5QcPHoytW7diwoQJGDx4MNLS0rBnzx40atRIa26qsbExGjVqhKNHj6JevXqwtLRE48aNyzQPkIihjCStW7duOHjwIDZt2oTjx49j9+7dMDIy0pxgVX1eLwBYtGgR6tati6CgIISEhMDGxgaTJk0qdUVhWdja2mLBggVYv349vvzySygUCmzfvh3W1tbo27cv7OzssGHDBmzatAlyuRz29vZo2bKlJiiuWLEC8fHx2LNnj2Z+j5GREZYsWYKhQ4fi+++/15x5f+7cufj666+xcuVK5OfnY8CAAW8Uyuzt7dG3b1/8/vvviI+Ph5OTE3bs2IH169fj2LFjCA4Ohrm5OerVq4fp06drTcwPCAiAq6sr9u/fj++//x4WFhbw9PTUDB03aNAAq1evxsqVK7F06VLY2Nhg+PDhsLKywv/7f//vjdq4rBo3boxdu3Zh+fLlWL9+PVQqFby9vfHDDz+8UTu96I8//sAff/xRYvuAAQPg4+ODmjVr4ueff8bSpUuxcuVKVK9eHf369YOfnx8mTJig9ZgOHTpg27ZtWLduHTZv3gyVSgVHR0etY/VtGRsbY+fOnVi3bh1CQkJw4MABVKtWDR4eHtiwYYNWD1KtWrXQp08fhIaG4uDBg5DJZGjQoAFWrlyJnj17au5XlmNaKP3794e+vj62bduGtLQ0eHt746uvvtI6qXTDhg2xdOlSrF69GgEBAWjUqBG+//57/PHHHyV6phctWoSFCxciICAAhYWFmDZtGkMZlYmeqjz7vImIiN4RDx48QLdu3TB79uwS4ZdICJxTRkRERCQCDGVEREREIsBQRkRERCQCnFNGREREJALsKSMiIiISAYYyIiIiIhGQ3HnKlEolioqKoK+vz8thEBERUYVTqVRQKpUwMDB45XVdJRfKioqKcP36daHLICIiIonx8vLSXA2jNJILZeqE6uXlpbnWHhEREVFFUSgUuH79+it7yQAJhjL1kKVMJmMoIyIiokrzumlTnOhPREREJAIMZUREREQiwFBGREREJAIMZUREREQiwFBGREREJAIMZUREREQiwFBGREREJAIMZUREREQiwFBGREREJAIMZUREREQiwFBGREREJAIMZUREREQiwFBGREREJAIMZUREREQiwFBGREREJAIMZUREREQiwFBGREREJAIMZUREREQiwFBGREREJAIMZUREREQiwFBGREREJAKChrJ//vkHkydPRvv27eHq6oqQkJDXPubSpUsYMGAAPD090aNHDxw4cKASKiUiIiKqWIKGstzcXLi6uuKbb74p0/0TEhIwadIk+Pr64vfff8eYMWMwb948nD17toIrJSIiIqpYBkLuvFOnTujUqVOZ779nzx7UrVsXc+fOBQA0bNgQV65cwdatW9GhQ4eKKpOIiIiowgkaynR19epV+Pn5aW1r3749vvvuO52fS6FQlFdZRERERC9V1szxToWy1NRU2NjYaG2zsbFBdnY28vPzYWxsXObnun79eolthoaGcHd3h4HBO9UsZVJUVIRbt26hsLBQ6FLeWc+ODw8YGMiELqXcFRUpcOvWTR4fVC4MDQ3h7uEOA1kV/CxVFOHWTX6WUsWoeu+YMvLy8oJMVvLLVSaT4Yd525FwL1mAqiqGY317fL5oNDw8PIQu5Z0nk8nw7aYg3E9KFbqUcuNc2wZfTxjA44PKlUwmw5LQvYh/+ljoUsqNU3U7zPUbyvcK6UyhUJTaGfSidyqU2djYIDVV+8swNTUV5ubmOvWSAc8+MEoLZQCQcC8ZsZEP3rhOsXrZ6yXd3E9KRXTCI6HLKHc8Pqi8xT99jJiMRKHLKHd8r1BFeafOU9asWTNcvHhRa9uFCxfQrFkzYQoiIiIiKieChrKcnBzcvn0bt2/fBgA8ePAAt2/fRmLis19Wy5cvx+zZszX3HzZsGBISEvD9998jNjYWu3btwtGjRzF27FghyiciIiIqN4IOX964cQOjR4/W/B0QEAAAGDBgAJYsWYKUlBQkJSVpbnd0dMT69esREBCA7du3o1atWli0aBFPh0FERETvPEFDma+vL6Kiol56+5IlS0p9THBwcAVWRURERFT53qk5ZURERERVFUMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJAEMZERERkQgwlBERERGJgOChbNeuXejatSu8vLwwZMgQREREvPL+W7duRc+ePeHt7Y1OnTrhu+++Q0FBQSVVS0RERFQxBA1lR44cQUBAAKZOnYqgoCC4ublhwoQJSEtLK/X+hw4dwvLlyzFt2jQcOXIEixcvxpEjR7BixYpKrpyIiIiofAkayrZs2YIPP/wQgwYNQqNGjbBgwQIYGxsjMDCw1PuHh4ejefPm6Nu3L+rWrYv27dvj/ffff23vGhEREZHYGQi1Y7lcjps3b2LSpEmabfr6+mjbti3Cw8NLfYyPjw8OHjyIiIgIeHt7IyEhAadPn8YHH3yg8/4VCkWp22Uymc7P9a542WumsuPxQVQ2fK8QPVfWY0awUJaRkQGFQgFra2ut7dbW1rh7926pj+nbty8yMjIwYsQIqFQqFBUVYdiwYZg8ebLO+79+/XqJbSYmJnB3d9f5ud4VUVFRyMvLE7qMdxaPD6Ky4XuF6M0IFsrexKVLl7B+/Xp888038Pb2Rnx8PBYvXox169Zh6tSpOj2Xl5dXlf4lVxpXV1ehSyAR4/FBVDZ8r5CuFApFqZ1BLxIslNWsWRMymazEpP60tDTY2NiU+phVq1ahX79+GDJkCIBnb4zc3Fx8/fXXmDJlCvT1yz5FTiaTSS6USe31km54fBCVDd8rVFEEm+hvZGQEDw8PhIaGarYplUqEhobCx8en1Mfk5+eXCF7qN4dKpaq4YomIiIgqmKDDl+PGjcOcOXPg6ekJb29vbNu2DXl5eRg4cCAAYPbs2bC3t8esWbMAAF26dMGWLVvg7u6uGb5ctWoVunTpwl8uRERE9E4TNJT17t0b6enpWL16NVJSUtCkSRNs3LhRM3yZlJSk1TM2ZcoU6OnpYeXKlUhOToaVlRW6dOmCmTNnCvUSiIiIiMqF4BP9/f394e/vX+ptO3bs0PrbwMAA06ZNw7Rp0yqjNCIiIqJKI/hlloiIiIiIoYyIiIhIFBjKiIiIiESAoYyIiIhIBBjKiIiIiESAoYyIiIhIBBjKiIiIiESAoYyIqBwolEqhS6gwVfm1EYmJ4CePJSKqCmT6+vji6H7cTU8VupRy1cDKBgHvDRa6DCJJYCgjIiond9NTEfk4SegyiOgdxeFLIiIiIhFgKCMiIiISAYYyIiIiIhFgKCMiIiISAYYyIiIiIhEo0+rL7OzsMj+hubn5GxdDREREJFVlCmUtW7aEnp7eK++jUqmgp6eH27dvl0thRERERFJSplC2ffv2iq6DiIiISNLKFMpat25d0XUQERERSVqZQllkZGSZn9DNze2NiyEiIiKSqjKFsv79+0NPTw8qleqV9+OcMiIiIqI3U6ZQdvz48Yqug4iIiEjSyhTKHBwcKroOIiIiIkkrUygrTUxMDBITE1FYWKi1vVu3bm9dFBEREZHU6BzKEhISMHXqVERHR2vNM1Ofx4xzyoiIiIh0p/NllhYvXoy6deviwoULMDY2xuHDh7Fz5054enpix44dFVEjERERUZWncygLDw/H//73P1hZWUFfXx96enpo2bIlPv30UyxatKgiaiQiIiKq8nQOZUqlEmZmZgCAmjVr4vHjxwCeLQa4d+9e+VZHREREJBE6zylr3LgxoqKi4OjoiKZNm2Ljxo0wNDTEvn374OjoWBE1EhEREVV5OveUTZkyBUqlEgDwv//9Dw8ePMDIkSNx+vRpfPnll+VeIBEREZEU6NxT1qFDB81/Ozs749ixY8jMzESNGjU0KzCJqOpTKJWQ6ev8u+6dUJVfGwlDqVJCX6/qHVNv+rpUKgX09GQVUJGw3vZ16RzKsrKyoFAoYGlpqdlmaWmJzMxMGBgYwNzc/I2LIaJ3h0xfH/P2HsC9x6lCl1Ku6tvZYNHQgUKXQVWMvp4+9tzajMe5SUKXUm7sTGtjmPv4N3qsnp4MF+8sxtO8+HKuSjjVTZzQpvHbjRjqHMpmzpyJLl26YOTIkVrbjx49ihMnTuCXX355q4KI6N1x73EqohIfCV0G0TvhcW4SErMThC5DNJ7mxSMz947QZYiKzn2OERERaNOmTYntrVu3RkRERLkURURERCQ1OocyuVyOoqKiEtuLioqQn59fLkURERERSY3OoczLywv79u0rsX3Pnj3w8PAol6KIiIiIpEbnOWWffPIJxo0bh8jISPj5+QEAQkNDcf36dWzevLncCyQiIiKSAp17ylq0aIG9e/eiVq1amsn9Tk5OOHjwIFq2bFkRNRIRERFVeTr3lAFAkyZNsHz58vKuhYiIiEiy3uhMdvHx8fjxxx8xa9YspKWlAQBOnz6NO3e4tJWIiIjoTegcysLCwtC3b19ERETgzz//RG5uLgAgKioKa9asKfcCiYiIiKRA51C2fPlyfPLJJ9iyZQsMDQ0129u0aYOrV6+WZ21EREREkqFzKIuOjkb37t1LbLeyskJGRka5FEVEREQkNTqHMgsLC6SkpJTYfvv2bdjb25dLUURERERSo3Mo69OnD5YtW4aUlBTo6elBqVTiypUrWLp0Kfr3718BJRIRERFVfTqHspkzZ6JBgwbo3LkzcnNz0adPH/j7+8PHxweTJ0/WuYBdu3aha9eu8PLywpAhQ157/cynT59iwYIFaN++PTw9PdGzZ0+cPn1a5/0SERERiYnO5ykzMjLCokWLMHXqVERHRyMnJwfu7u6oV6+ezjs/cuQIAgICsGDBAjRt2hTbtm3DhAkTcOzYMVhbW5e4v1wux7hx42BtbY1Vq1bB3t4eiYmJqF69us77JiIiIhKTNzp5LADUrl0btWvX1vz9119/Yc2aNTh06FCZn2PLli348MMPMWjQIADAggULcOrUKQQGBmLixIkl7h8YGIgnT55gz549mpWfdevWfdOXQERERCQaOoWyPXv24MKFCzA0NMTo0aPRtGlThIaGYunSpYiLi8MHH3xQ5ueSy+W4efMmJk2apNmmr6+Ptm3bIjw8vNTHnDhxAs2aNcO3336L48ePw8rKCu+//z4+/vhjyGQyXV4KFApFqdt1fZ53ycteM5Udj4/nqnJbAGyPF7E9nnuTz1K2hzaptUdZ26jMoWzDhg1YvXo1XFxccO/ePRw/fhyTJ0/Gzp07MXr0aAwdOhQ1atQoc9EZGRlQKBQlhimtra1x9+7dUh+TkJCAixcvom/fvtiwYQPi4+OxYMECFBUVYdq0aWXeNwBcv369xDYTExO4u7vr9DzvkqioKOTl5QldxjuLx8dzVb0tALbHi9gez+n6Wcr20Mb2eLkyh7LAwEAsXLgQAwYMwOXLl+Hv74/w8HD89ddfMDU1faOd60qlUsHa2hoLFy6ETCaDp6cnkpOTsWnTJp1DmZeXV5VO6qVxdXUVugQSMR4f2tge2tgez7EttLE9tJXWHgqFotTOoBeVOZQlJSWhTZs2AICWLVvCwMAA06dPf+NAVrNmTchkMs21M9XS0tJgY2NT6mNsbW1hYGCgFaYaNGiAlJQUyOVyGBkZlXn/MplMcqFMaq+XdMPjQxvbQxvb4zm2hTa2h7a3aY8ynxJDLpejWrVqmr8NDQ11Gq58kZGRETw8PBAaGqrZplQqERoaCh8fn1If07x5c8THx0OpVGq2xcXFwdbWVqdARkRERCQ2Ok30X7lyJUxMTAAAhYWF+Omnn2BhYaF1ny+++KLMzzdu3DjMmTMHnp6e8Pb2xrZt25CXl4eBAwcCAGbPng17e3vMmjULADB8+HDs3LkTixcvhr+/P+7fv4/169dj1KhRurwMIiIiItEpcyhr1aoV7t27p/nbx8cHCQkJWvfR09PTaee9e/dGeno6Vq9ejZSUFDRp0gQbN27UDF8mJSVBX/95Z17t2rWxadMmBAQEoF+/frC3t8fo0aPx8ccf67RfIiIiIrEpcyjbsWNHhRTg7+8Pf3//Mu/Tx8cH+/btq5BaiIiIiISi82WWiIiIiKj8MZQRERERiQBDGREREZEIMJQRERERiYDOoSwxMREqlarEdpVKhcTExHIpioiIiEhqdA5l3bp1Q3p6eontmZmZ6NatW7kURURERCQ1OocylUpV6vnIcnNztc74T0RERERlV+bzlAUEBAB4doLY4mf2B55daDMiIgJubm7lXyERERGRBJQ5lN26dQvAs56y6OhoGBoaam4zMjKCm5sbxo8fX/4VEhEREUmAzmf0/+KLL/Dll1/C3Ny8wooiIiIikhqd55R9/vnnLw1kUVFRb10QERERkRTpHMr69u2LU6dOldi+adMmDBkypDxqIiIiIpIcnUPZ2LFjMX36dHzzzTfIz89HcnIyxowZg40bN2L58uUVUSMRERFRlVfmOWVqH3/8Mdq1a4fZs2ejX79+ePLkCby9vXHw4EHY2tpWRI1EREREVd4bXWbJyckJjRs3xsOHD5GdnY3evXszkBERERG9BZ1D2ZUrV9CvXz/cv38fBw8exPz587Fw4UJ88sknePLkSUXUSERERFTl6RzKxowZg969e2Pv3r1o2LAhhgwZguDgYCQlJaFv374VUSMRERFRlafznLLNmzejdevWWtucnJywe/du/PTTT+VWGBEREZGU6NxTpg5k9+/fx9mzZ5Gfnw/g2eWXpk6dWr7VEREREUmEzj1lGRkZ+OSTT3Dp0iXo6enhr7/+gqOjI/7f//t/sLS0xJw5cyqiTiIiIqIqTeeesoCAABgYGODUqVMwNjbWbO/duzfOnDlTrsURERERSYXOPWXnz5/Hpk2bUKtWLa3t9erVQ2JiYrkVRuKgVCqhr/9GZ04Rtar6uoiI6N2lcyjLzc3V6iFTy8zMhJGRUbkUReKhr6+P7388gPgHKUKXUm6c6tpi9syBQpdBRESkRedQ1rJlSwQHB+OTTz7RbFMqldi4cSN8fX3LszYSifgHKYi9+0joMoiIiKo0nUPZ559/jrFjx+LGjRsoLCzEDz/8gJiYGDx58gS7d++uiBqJiIiIqjydQ5mLiwv+/PNP7Ny5E2ZmZsjNzUWPHj0wcuRI2NnZVUSNRERERFWezqEsMTERtWvXxpQpU0q9rU6dOuVSGBEREZGU6Lz8rFu3bkhPTy+xPSMjA926dSuXooiIiIikRudQplKpoKenV2J7bm4uqlWrVi5FEREREUlNmYcvAwICADy7nNLKlSthYmKiuU2hUCAiIgJubm7lXyERERGRBJQ5lN26dQvAs56y6OhoGBoaam4zMjKCm5sbxo8fX/4VEhEREUlAmUPZjh07AABffPEFvvzyS5ibm1dYUURERERSo/PqS/UwJhERERGVnze6zNKGDRtw8eJFpKWlQalUat1+/PjxciuOiIiISCp0DmXz5s1DWFgYPvjgA9ja2pa6EpOIiIiIdKNzKDtz5gzWr1+PFi1aVEQ9RERERJKk83nKqlevDktLywoohYiIiEi6dA5lM2bMwKpVq5CXl1cR9RARERFJks7Dl1u2bEF8fDzatm2LunXrwsBA+ymCgoLKrTgiIiIiqdA5lHXv3r0i6iAiIiKSNJ1D2bRp0yqiDiIiIiJJ0zmUqd24cQOxsbEAgMaNG8Pd3b3ciiIiIiKSGp1DWVpaGmbOnImwsDBUr14dAPD06VP4+vrixx9/hJWVVbkXSURERFTV6bz6cuHChcjJycHhw4cRFhaGsLAw/PHHH8jOzsaiRYsqokYiIiKiKk/nUHb27Fl88803aNiwoWZbo0aN8M033+DMmTPlWhwRERGRVOgcypRKJQwNDUtsNzAwKHEdzLLatWsXunbtCi8vLwwZMgQRERFletzhw4fh6uqK//73v2+0XyIiIiKx0DmUtWnTBosXL0ZycrJmW3JyMgICAuDn56dzAUeOHEFAQACmTp2KoKAguLm5YcKECUhLS3vl4x48eIClS5eiZcuWOu+TiIiISGx0DmVff/01srOz0a1bN3Tv3h3du3dHt27dkJ2dja+++krnArZs2YIPP/wQgwYNQqNGjbBgwQIYGxsjMDDwpY9RKBT47LPPMH36dDg6Ouq8TyIiIiKx0Xn1Ze3atREUFIQLFy7g7t27AICGDRuibdu2Ou9cLpfj5s2bmDRpkmabvr4+2rZti/Dw8Jc+bt26dbC2tsaQIUNw5coVnfcLPAt2pZHJZG/0fO+Cl73mV2F7aGN7PFeV2wJge7yI7fEcPzu0sT20ldYeZW2jNzpPmZ6eHtq1a4d27dq9ycM1MjIyoFAoYG1trbXd2tpaE/hedPnyZezfvx/BwcFvte/r16+X2GZiYlKlz7cWFRWl0zVL2R7a2B7PVfW2ANgeL2J7PMfPDm1sD226tkdxZQ5loaGhWLhwIfbt2wdzc3Ot27KysjBs2DAsWLCgQud4ZWdnY/bs2Vi4cOFbnw/Ny8urSif10ri6ugpdgqiwPbSxPbSxPbSxPZ5jW2hje2grrT0UCkWpnUEvKnMo27ZtGz788MMSgQwALCwsMHToUGzZskWnUFazZk3IZLISk/rT0tJgY2NT4v4JCQl4+PAhpkyZotmmXvHp7u6OY8eOwcnJqUz7lslkkgtlUnu9r8P20Mb20Mb20Mb2eI5toY3toe1t2qPME/2joqLQoUOHl97erl073Lx5U6edGxkZwcPDA6GhoZptSqUSoaGh8PHxKXH/Bg0a4NChQwgODtb869q1K3x9fREcHIxatWrptH8iIiIisShzT1lqaioMDF5+dwMDA6Snp+tcwLhx4zBnzhx4enrC29sb27ZtQ15eHgYOHAgAmD17Nuzt7TFr1ixUq1YNLi4uWo9XX+rpxe1ERERE75IyhzJ7e3vcuXMHzs7Opd4eFRUFW1tbnQvo3bs30tPTsXr1aqSkpKBJkybYuHGjZvgyKSkJ+vo6n7mDiIiI6J1S5lDWqVMnrFq1Ch06dEC1atW0bsvPz8eaNWvQpUuXNyrC398f/v7+pd62Y8eOVz52yZIlb7RPIiIiIjEpcyibMmUK/vrrL/Ts2RMjR45E/fr1AQB3797Fr7/+CoVCgcmTJ1dYoURERERVWZlDmY2NDfbs2YP58+djxYoVUKlUAJ6ds6x9+/b4+uuvS10xSURERESvp9PJYx0cHPDLL7/gyZMnuH//PgDA2dkZNWrUqJDiiIiIiKTijc7oX6NGDXh7e5d3LURERESSxWWNRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCIgilC2a9cudO3aFV5eXhgyZAgiIiJeet99+/ZhxIgRaNWqFVq1aoWxY8e+8v5ERERE7wLBQ9mRI0cQEBCAqVOnIigoCG5ubpgwYQLS0tJKvf+lS5fQp08fbN++HXv27EHt2rUxfvx4JCcnV3LlREREROVH8FC2ZcsWfPjhhxg0aBAaNWqEBQsWwNjYGIGBgaXef/ny5Rg5ciSaNGmChg0bYtGiRVAqlQgNDa3kyomIiIjKj4GQO5fL5bh58yYmTZqk2aavr4+2bdsiPDy8TM+Rl5eHoqIi1KhRQ6d9KxSKUrfLZDKdnudd8rLX/CpsD21sj+eqclsAbI8XsT2e42eHNraHttLao6xtJGgoy8jIgEKhgLW1tdZ2a2tr3L17t0zPsWzZMtjZ2aFt27Y67fv69esltpmYmMDd3V2n53mXREVFIS8vr8z3Z3toY3s8V9XbAmB7vIjt8Rw/O7SxPbTp2h7FCRrK3taGDRtw5MgRbN++HdWqVdPpsV5eXlU6qZfG1dVV6BJEhe2hje2hje2hje3xHNtCG9tDW2ntoVAoSu0MepGgoaxmzZqQyWQlJvWnpaXBxsbmlY/dtGkTNmzYgC1btsDNzU3nfctkMsmFMqm93tdhe2hje2hje2hjezzHttDG9tD2Nu0h6ER/IyMjeHh4aE3SV0/a9/HxeenjfvnlF/zf//0fNm7cCC8vr8oolYiIiKhCCT58OW7cOMyZMweenp7w9vbGtm3bkJeXh4EDBwIAZs+eDXt7e8yaNQvAsyHL1atXY/ny5XBwcEBKSgoAwNTUFGZmZoK9DiIiIqK3IXgo6927N9LT07F69WqkpKSgSZMm2Lhxo2b4MikpCfr6zzv09uzZg8LCQvzvf//Tep5p06Zh+vTplVo7ERERUXkRPJQBgL+/P/z9/Uu9bceOHVp/nzhxojJKIiIiIqpUgp88loiIiIgYyoiIiIhEgaGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAREEcp27dqFrl27wsvLC0OGDEFERMQr73/06FH06tULXl5e6Nu3L06fPl1JlRIRERFVDMFD2ZEjRxAQEICpU6ciKCgIbm5umDBhAtLS0kq9/7///otZs2Zh8ODBCA4ORrdu3TB16lRER0dXcuVERERE5UfwULZlyxZ8+OGHGDRoEBo1aoQFCxbA2NgYgYGBpd5/+/bt6NChAz766CM0bNgQn3zyCdzd3bFz585KrpyIiIio/BgIuXO5XI6bN29i0qRJmm36+vpo27YtwsPDS33M1atXMXbsWK1t7du3R0hISJn2qVKpNPuWyWQlbpfJZKjXuDYMjUre9q5ycLaDQqGAQqHQ+bEymQz1ne1gaCB4fi83dR1s3qo9GjrYVqn2cLK3fqP2kMlkaGxvCyNZ1WkLAHC2efP2cLG2hZF+1WqPejXfvD3qV7eHoV7V+Syta/F2nx21TBwgQ9VpD1uTWm/VHtWN60NPZVgBlQnDwrjuS9tDvU2dQV5G0FCWkZEBhUIBa2trre3W1ta4e/duqY9JTU2FjY1NifunpqaWaZ9KpRIAcOvWrZfep1PfJgCalOn53hVXr15948d2bucEwKncahGDt2mPnt6OABzLrRYxeNP26NvQCWhYtY4N4M3bo79tPcC2XnmWIgpv2h5djRoC1g3LtxiBvc1nR0N4oaGeV/kVI7T8t2sPQ/SAVflVI7ys17eHOoO8jKChTAgGBgbw8vKCvr4+9PT0hC6HiIiIqjiVSgWlUgkDg1fHLkFDWc2aNSGTyUpM6k9LSyvRG6ZmY2NTolfsVfd/kb6+PoyMjN6sYCIiIqIKIujkByMjI3h4eCA0NFSzTalUIjQ0FD4+PqU+plmzZrh48aLWtgsXLqBZs2YVWSoRERFRhRJ8Ruq4ceOwb98+BAUFITY2FvPnz0deXh4GDhwIAJg9ezaWL1+uuf/o0aNx9uxZbN68GbGxsVizZg1u3LgBf39/oV4CERER0VsTfE5Z7969kZ6ejtWrVyMlJQVNmjTBxo0bNcORSUlJ0C+2mql58+ZYtmwZVq5ciRUrVqBevXpYt24dXFxchHoJRERERG9NT/W69ZlEREREVOEEH74kIiIiIoYyIiIiIlFgKCMiIiISAYYyIiIiIhFgKCMiIiISAYaySlRUVCR0CURERFXau3xSCYaySqA+QI4cOYLMzExhixEJhUKB/Px8ocsQjYSEBPz+++9ClyE49XslPT0dUVFRAlcjPHV75OTkCFyJuOTm5gpdAomM+kLfBQUF0NPTe2eDGUNZJdDT00NhYSFu3boFS0tLocsRlPqNcu3aNRw/flzgasTjwYMHMDU1FboMwenp6QEAIiIicPLkSQDPP2ylSN0e48aNQ3p6usDViMe+ffsk/6NO/Vl69uxZ/PLLLwJXIzz1e2XWrFm4cOGC5u93DU8eWwlUKtU7e4BUlHPnzsHCwgJNmzaVfPsolUro6+tLvh2Kk8vlUCqVMDY2FroUUUhNTdVc5UTqsrKysHv3bkycOFHoUkQhNjYWjx8/hp+fHxQKBWQymdAlCSo9PR01atR4Z9uBoayCqb9w7969C0dHRxgaGgpdEolMUVER0tLSYG9vL3QpoqF+3xA0X7RsE/7ApVerCu+Rd7v6d4D6ANm1axdSU1MFrkZYxecLRUZGClyNeFy7dg3/93//J3QZglMfH1FRUdi9e7fA1QhP3R6HDx8ucQ1gKVIHsosXLyIvL0/ockQhJycHoaGhQpchOPUUh7CwMERHRwtczduR9ru8kiiVSgwZMgS1a9cWuhRBFZ8vdOrUKQDSni+k5u7ujhkzZghdhuDUx4dCodAM1Un5+FC3R3x8PKytrQWuRnjq9ggJCZF8QFW7efMm7t69C+DdXnH4ttTHw7///vvOHxscvqRKJ+X5Qhx+obLisVJSUVERkpKS4OjoKHQpopCTk4P8/HxYW1vzeKki3u1IKVLqnKtQKAAAu3fvRlJSkpAlVbpXZX0DAwNJBjIAJT40MzIysH//foGqER+5XI7Tp08LXYbglEol9PT0kJKSAkDavSDFGRgYMJAVY2ZmpulFlWogU7830tLSkJ2dLXA1b4+hrAKo3xzq1R/x8fGoVq2akCVVGvUbpPhQlFKp1AxDSXm+UGhoKBISErS2VZUPkvJy9+5dxMTEAJB2EFEPwSxbtgyRkZGS/cJV/7BVHwvfffcdHj9+LGRJglN/lt69exfz588XthiBlPbZcOTIEZw/f16AasoXQ1k5KH6A3L9/H6dOncKjR4802+bMmQMrKyshSqt0enp6+PPPP3H//n0Az4Kpvr6+5ktGLpdLbr5QVlYWAGDr1q2aHkL1l03NmjUxduxYoUoTnEqlQmZmpmbidoMGDdC/f39hixKR6dOnw83NTegyBKP+YaunpwelUgkHBwfY2dkJXFXl+eeff5Cfn4/s7GzN56X6s9TMzAy9e/cGIJ3PUjU9PT1ER0cjPz9f84OlQ4cOaN++vcCVvT0DoQuoCoqKimBoaIiVK1ciOjoaKpUK2dnZqF69OmrVqoWhQ4fCxcVF6DIrhUKhwL59+xAfHw9TU1P4+vqiRYsW8PX1haWlJby8vODl5QUA7/yEzLLIz8/HyZMnERYWhjt37iAhIQF6enqaYLpw4UIsXLgQFhYWAldauYqKivD9998jNDQUnp6eaNCgARo1aoS6deuicePGAKQ7HFNc3bp1hS6hUqlPaZCeno7ffvsN165dw+TJk+Ht7Q19fX2MGTOmSpz2oKxOnTqFVq1aYfXq1TAzM4OzszPq1q2L2rVrw97eXnMaHSm0h/r/e2xsLI4fP47Tp0+jYcOG+Pbbb5GUlIQ6derAyMhI6DLfGkNZOVCfe+z48eOYM2cOHBwckJaWhnv37uHKlSuSuualTCbDpk2bEBkZiU2bNuH06dOIiIjA0qVL4eDggObNm2PmzJlCl1lplEol7OzskJSUBIVCgeDgYBQVFcHe3h55eXmIi4uTVCBTf7CeOnUKt2/fxldffYXr16/j1q1bOHnyJGrVqoUVK1YIXaYoSHni9hdffIFu3brhypUryMjIAPCsp7lLly5wdnYWuLrKM378eBQWFiIvLw9JSUm4fv06TExMYG1tDQcHBwwbNkwyx4h6RGrv3r2wsbGBk5OTptf0zJkzMDMzw/vvvy9kieWCoewtJScn4/79+7C2toa7u7um+7R+/frw9vZG165dJbWcXX2iy+PHj8PBwQEzZ86ElZUV/vrrL2zdulVrEcS7esZlXZiamqJNmzaoUaMG0tPTYWBggNu3byM5ORkGBgb473//K3SJlUr9iz45ORm9evVC69at0bp1awBAZmamZthfSr0hcrkcISEh6NWrl9ZrlsqXbXH6+vooKChAWloaPvzwQxw4cABNmzYFABw8eBBdu3YVuMLKpf7umDNnDgoLCxEbG4vIyEhcv34djx8/1lzjUQrHivq9ERERgT179mDWrFlo0aIFAOD06dPo2bOnkOWVG4ayt3Tjxg1s2rQJNjY2uHfvHr777jsMGDAAjRs3hpGRkaQCGfD8iyQsLAyzZs1CnTp1AAD9+vXDrVu34OvrC0Aa3e3FNWnSRPPfvr6+yMnJgaGhYZXobn8THTt2xNatW3H+/Hk0bNgQVlZWsLS0hKWlJVQqlSSOj/z8fBgbG+P69es4deoUevfurfmCzcnJQWZmJhwcHIQus9LdvHkTbm5uyMjIgLGxMSwtLZGYmAgAcHJyEri6yqf+IWtoaAg3Nze4ubmhf//+kMvlAldWudTvC0dHR1y5cgWxsbHw8/MDADx8+LBKzCcDGMrempeXFz799FMkJSXhzp07iI6OxurVq2FoaAhzc3OMGTMGrq6uQpdZadTXcGzevDnWrl2LKVOmwNnZGVZWVjh37hwGDx4sdImCKT5R18zMTOBqhJOZmYmFCxfixo0bePz4MRwcHODk5IRatWqhU6dOkuhBBZ5ds/D333/HhQsX4OLigpSUFFhYWMDY2BhhYWEIDw/Hp59+KnSZlc7FxQUmJiYYOXIkvLy8IJfLcfDgQc28XKn0sqsV/5GiUqk0f6t/0Emhlwx49trNzMwwYMAArF+/HhYWFrh06RJ+//131KtXr8p0gDCUvSU7OzvY2dnh9OnTmDhxItLS0pCYmIiHDx8iKipKkl++enp6GDt2LNasWYOgoCAUFRXh5s2baN68ORo1aqS5j9RIoffnVdRDksePH4exsTH+/vtvhIaG4tatW7hw4QKsra0lNTxlY2MDX19f7Nu3D3Xq1MH8+fMhk8ng6OiI8+fPY8KECUKXKAhzc3MMHz4c+fn5ePr0KXr16oWuXbti8uTJAKT32aH+3FAqlZIZqiyN+nV7enqiU6dOOHz4MDZv3ow2bdrgP//5j8DVlR+e0f8tyOVyGBgYIDQ0FFu2bMHGjRs1txUUFOD27dto1qyZcAUKLDk5GaGhoVAoFGjcuDEaNmwIMzMzSX6wFBQUIDU1Ffb29jAwkOZvIXUo+/nnn1GrVq0Sp75IS0uDtbW1pOaTKZVKREREwNraGvHx8UhISNCc13DChAkwNzcXusRKV1RUBAMDA0RGRiItLQ1ubm4wNzeXzLkegee9gSdPnoRKpYKfnx9MTEy07iPFz9EXPX36FNWrVxe6jHIlzW+HcnL37l2cP38eZ8+ehZ2dHTIzM1FQUAB7e3tcvHgR165dk3Qos7e3L/WcU1L7INmzZw+OHj0KMzMzLFu2TPOlI7Vwpg5aqamp+Ouvv1BUVAR3d3fUqlULVlZWmkvFSCGQqb90L1++jLS0NDRr1kzSZ6pXB/GEhAT8+uuvOHz4MPz8/ODp6YmmTZtKKpABz8/PduHCBezZs0czItOxY0d06NABnp6ekvscVVOfjNzAwKDKBTKAJ499K1ZWVjAyMsK9e/cQHR2NefPmYePGjQgODsbPP/9cJQ8YXajnP0iR+nWHhYXh1KlTGDRoEB49egRTU1PcuXMHy5cvF7hCYeTk5MDU1BSurq44e/YsduzYgc2bN+PXX3+V5C//wMBAzRUd1q1bh6ZNm2LFihUoKCgQuLLKpX6/LFmyBEZGRli1ahVatWqF4OBgfPPNNygsLBS4wsqXn5+PgoICzJo1C0uXLkWfPn2wd+9ezJ49G7Nnz9aclFpq9PX1q/QP2qr7yiqBnZ0dRo0ahdq1a6N58+YIDw/H5cuXERISgp49e2LgwIFCl1hp1F+oqampqF69OoyMjKBUKiU1Ibc49Wu/ePEiunfvjho1asDd3R3Asx7W+Ph4gSsUhpmZGT799FMUFRUhJiZGs7w/JSVFUsv71e+LqKgoLF26FBcuXMD9+/fxyy+/YPPmzUhKSkK9evWELbISFf+cGDFiBOzt7eHj44PBgwdj2LBhiIqKgqenp4AVVp7i5/J7/Pgxvv32WxQVFaF58+YwNTVFXFwcMjMzsW/fPsnNO5TCNJCq+aoqifrN0717dwBAt27d0K1bNwDSWyGkduDAAcjlckybNk2Sr19N/drz8vLQpEkT7Ny5U7Py9MqVK2jXrp2Q5QlG3SNiYGAg6eX9wLM5l4aGhti/fz+OHTuGsWPHonXr1liwYIHmVDJSoA7iiYmJUKlUiIiIQI8ePQA8+xLOzs6WVEBV/yjJzc2FkZERsrKyYG5urtmup6eHnj174rfffhOyzEonlWkgVevVVDJ9fX1NMFMP1alUKshkMskFEvUHRuvWrbFkyRJcvnwZM2bMgI+Pj8CVCWvEiBGYNWsWbty4AT8/P9y/fx937tzBpEmThC5NEFze/4xSqYS9vT38/f1x/PhxNG7cGO3bt8e5c+dgbW0tqfPXqf+fx8XFITQ0FGfOnIGXlxc8PT1RUFCAYcOGSWrBg7o9OnfujJMnT2Lx4sVo0aIFEhIScPnyZcyaNQuhoaGaE6dWZerAXnwayNatW2Fqaorr16/jyJEjmDNnjtBlliuuvixHUhl6eR25XI6tW7ciLS0NAwcO1JynTSrtow7q165dQ0ZGBpycnLB//36EhYWhRo0amDNnjmSuhfoyxZf3S2Fif3Hq163+pa9WWFiIvXv3wtTUVFJTH4pTqVSIjIzEuXPncObMGdy7dw/6+vpo1KgR/ve//0lu4VRGRgb27Nmj6UVs06YNevTogenTp+Orr76q8otD1CNOq1evRp06dWBra4u///4bixYtwu+//46//voL69atE7rMcsVQ9hZyc3Px+PFjSXWtv05OTg7MzMzw4MEDzJgxA3fv3sVXX30lqS8ZdShbunQpateujdGjRyMvL09rSbtUAirA5f0v07NnTzg4OKBNmzZo164dPDw8AEh36kNubi6Sk5Ohr68PW1tbmJqaIjs7Gzdv3sThw4cxaNAgzSWXpODBgwd4+vQpGjdurLm+slpycrLmYuRSsHTpUrz//vuaaSAtWrTA119/DTc3N4wYMULo8soVhy/fgPpL9+jRowgNDcWyZcsQGRmJP/74A61bt0bHjh2FLrFSqdtj165diI2Nxfnz5+Hp6YlmzZrB1tYW27Ztw/379zFlyhQYGxsLXW6FU/f85OTkwNLSEgAkG8gALu8vjVKpxA8//ICIiAj8888/CAwMBAC0aNEC3333ncDVVR71Z0dcXBw2b96Mo0ePas7O3r9/f/Tq1Qu+vr6ay7NVdepA/n//93+IjIxETk4OCgoK4OjoCEdHR/Tv3x916tSBvb29pD5HpDQNhKHsDag7F8+cOYNBgwbhyZMn2LVrF6KjoxEREYGaNWvCy8tL4CorjzqExMXFwcfHByNHjkRGRgY8PT1hbGyMJ0+eYOzYsejdu7dkLjmVnZ2NGzduIDAwEIcPH0aHDh3g5+eHhg0bSuaDtLjiy/s9PT0RGRmJjRs34tChQ/D09MRXX30FCwsLocusFOp5dN7e3vDy8sLw4cMRHh6O7du3a654IRXqz9Jff/0V+vr6+Oeff/Do0SMEBwdj586d8PDwqPJDdMWpf8AcOnQIn332GRwcHJCamoqYmBhcvnwZXbt2RZ06dSRxguUXp4EsWbIE+/fvx/bt21GjRg0sWLCgSl4blqHsDajfOA8ePED9+vURGBgIBwcHLFy4EFOmTEFubq7AFQrjyy+/RGFhYYmu9ho1aqBOnTqoX7++QJVVPkNDQyxfvhwPHjzA1atXcenSJezcuRNNmjTBjz/+KHR5lYbL+7WpezdSUlJga2sLPT09yGQytGzZEiEhIZILZepgYWJigp49ewIAatWqhcmTJyMiIgIXL16Eo6OjJEKI+thISEhAs2bNNCv5AaB58+Z47733NEOWVb0tijt27Bhq166Nzp07Y/r06VV+1IGh7A3J5XJ07twZn3/+OTIyMrB7927k5uYiMTERLVu2FLq8SqPubg8PD8elS5eQmJgIc3NzuLq6av4VFRVh3rx5kllRplKpUK1aNTg6OqJ+/fro0KEDFAoF4uLiJHcSTC7v16anp4eCggKMGzcONWrUgIeHB9q0aQMPDw+cPHkSQ4cOFbrESqWnp4e8vDycPHlSc2UDmUyG3NxcKBQKyQxbAs8DRlRUFK5cuYINGzagd+/eqFu3LkxNTWFqaip0iZVKqtNAONFfR+qJ7Grh4eGoXr06GjZsiJCQEAQFBVW51SCvov4Fqz6JrqurK7KyspCamorExERMnDgRbdq0EbrMSnf58mWEhYUhIyMDDg4OaNWqFerVqyfJC9QDQHp6Or755huYmZmVury/Ro0aGDVqlNBlVgq5XI7IyEjExsbi6tWruHnzJh49eoTBgwfjk08+Ebq8SpeSkoKFCxfi+vXrSEtLg6OjI0xMTNCsWTPMmzdP6PIq3alTp3D48GE8fvwYxsbGsLCwgK2tLUaMGCGpoVzg2TSQ0aNHIyoqCu3bt9eaBlJVMZTpaOLEifjuu+/wxx9/wNHRER06dND0AKWlpSE7OxvOzs4CV1m58vLyMHHiROzYsQPAsy/g5ORkxMXFoUOHDjA3N6+yv2qKU7/G6OhoLFiwAG5ubjh37hy8vb0RERGBevXq4eeff67y7fAyUl/eX9yL7we5XK7pYZWyyMhInD17FleuXEFMTAySk5MxceJETJ8+XejSKo169CE9PR2RkZG4c+cObt++jenTp1fJOVSvUlBQgMTERM00kOjoaNy5c6dKTwNhKNNRZmYmLC0t8fXXX+PixYvIzMyEg4MDOnfujM6dO0tqybb6i+X27dtYuXIlPvvsMzRu3FjosgSj/jDdsGEDTExM0KhRI/z222/4/PPPsWjRInTs2FFyw1NqXN7/3JkzZxASEoKMjAw0a9ZMM3wpZTdv3oS9vT1sbGw023Jzc3H16lWYm5vD29tbwOoqj1KpRGxsLIKDg1GvXj1069YNVlZWknuPAKWfz6/4NBA3NzeBK6wYDGVvQaVSIT4+HmFhYThx4gTi4uJw9OhRocuqNOo3zZEjR/DTTz/B1NQUbdq0Qd26deHs7AxPT09JzYNQt8fEiRMxbdo0HD58GE2aNEH//v3x448/wsbGRjJDdEDZl/cDVXd+yIuSkpLw4YcfYuLEiSgqKsK1a9cQHx+PvLw8HDhwoMT526q6zMxMbN68GXFxcYiJiYGVlRVat24Nb29vdO7cWejyKo36vbJ9+3aEhobC0tISd+7cwcOHDzF79mwMGDBAMu+R4qQ4DYQT/d+Cnp4enJ2d4ezsjCFDhqCoqEjokiqV+gPCy8sLX331Fe7du4f79+/j6tWr+PvvvzFjxgxJ9QCo26N///5wdHSEra0tTpw4AXNzc5w4cQJLly4VuMLKxeX9z6m/UK9du4YePXpg1KhRyM3NxcCBA5Geno60tDRJBTL1//O///4bqampaN68OfLy8tC6dWtNSOvcubNkgoj6+P/rr78wbdo0zTzcc+fOYe/evWjRogWcnJyELLHSFJ8G8uOPP2pNA9m9e3eVnwbCUFaOqtqFUctK3fPRunVrFBUVISkpCTExMZIdyuzduzcAYOjQoUhMTMShQ4fg6+sLd3d3gSurPFzer039BeLs7Ix79+4BgKYXuUaNGpI6XQzw/PxkFy9exLhx43D27Fn4+vrio48+QkpKCtq1a6e5X1X98i1OT08POTk5yMnJ0VoY1b59e6xYsUJScw2VSiVkMhlOnTqFXr16oVGjRsjIyMCnn36qmQZSlY8JaaYIemvqX7oPHz5ESEgIEhISYG1tjUaNGqFRo0bw9fWVzCkwXsbCwgJTpkxBYWGhZphOKri8v3Q3btzA7t27sXv3bvj5+aFDhw5o06aN1lwqKVD3omZlZcHW1hYxMTEYMmQIACAmJgZdunQBIJ0L1CsUChgYGKBFixbw9/fHpEmT4OzsjLS0NBQVFUlqPpn6R9rly5c100A6duyI2rVro1GjRpDL5QJXWLE4p0xHJ0+exL179zB+/HihSxGUeg7E3LlzUatWLRw+fBi1atVCeno6DAwM8Omnn6JTp05Cl0kC4/L+5wH11q1bmDhxIlauXIns7GxcvnwZ//77L5KTk/H3339LosfwRQ8fPoSVlRWOHTuGtWvXwsPDA/fu3UNwcLAkr/+ZlpaGzZs3Iz09Hbdv34aFhQUmTJiAzp07S+6aqEeOHIGfnx8CAwMRERGBfv36YdWqVVi6dGmVHnVgT1kZ3b9/H3Z2djh58qRmJZB6VUhYWBjMzc2r9IHyIvWHQ1RUFJYsWYKIiAjMnz8fjx8/xsqVK1G3bl0A0hl+oNJ16NABnTt3LrG8X4oBJCUlBePHj0fLli1RWFiIjh07Ii8vD9nZ2ZJsDwCaUzwMGDAAxsbGePjwIYYPHw6ZTCaZz465c+fCy8sLfn5+aNCgAT7//HPExsbCxMQEpqammhOnSimQAdKdBsJQVkbh4eFYsGAB8vPz8fDhQ5iamqJVq1awtbXF6tWrMXz48Cp/sLwoJiYGNWrUQHZ2NtLS0uDk5AQnJydUq1YN9erVAyCd4QeAvagvUiqVuHv3rtby/rZt20pueb96jszff/+Np0+f4tGjR6hVqxYAwMzMrEqvJCuNOmzFxcUhKCgIhoaG+M9//oP33nuvxH2quuzsbNja2uLSpUsIDAyEkZERPD090a5dO3h5eWkCmZRJbRoIhy918OTJE8yaNQv16tXDiRMnkJiYCGdnZ9SpUwdr166V3Idramoq7t27h/r162PJkiXw9vbGkydPNNd5lMKqOuB5L2pAQAC8vb0xePBgSfeicnl/6QICAnDixAk8fvwYDRo0QIcOHdCxY0dJXZZNraioCAMGDICvry9iYmJw9epVyGQyODk5YfPmzahZs6bQJVYauVyOnJwcpKSkICYmBhEREbhz5w4SEhLQunVrLFq0SOgSqRIxlJWR+kLbBQUFmpUwBQUFCAsLg52dHVxdXQWuUFinTp3Chg0b4OLigm7dummu9yiFLvfg4GBNL2rbtm0xaNAgTS+qv78/hg8fjj59+ghdZqVRBy5/f/9Sl/d//vnnklneX5r79+8jLCwMJ0+eRFxcHI4cOSJ0SZXu5s2b+OGHH7B161bNtlu3buHs2bOYNGmScIUJTKVSQS6X4+nTp4iPj4eRkRG8vLwk81lKDGVlov6SKSgowJEjR5CbmwtfX180atRI6NIElZWVhXPnzsHMzAwtW7aEkZGRZE8Lwl5UbTk5OfD390dQUJDW9oEDB+Knn36S1PDlqxQ/W7kUbNu2DaampjAzM0NKSgrGjBkjdEmiIJVRhbKQ+jQQ6XwavAX1MMv06dPh4uKCHTt2wMTEBBYWFnB0dMQPP/wAa2trgausHOoPj2vXruG7776Dg4MDCgsLsW3bNtjY2MDT01NSZ60HnvWi1qhRA+vWrUO1atUwb948rV5UqQUyLu8vOykFMgDIz8/H/v37kZGRgczMTFy+fBmDBw9GmzZtJHUurhcxkHExnZq0PhHegLqX7MaNG1AqlRg6dCj+/fdfrF69GnPnzkW1atUkE8iA5yd9DAkJQevWrTF27FgkJSXh0aNHWqvqpPLLT6VSaYa1X+xF7dChg9DlCUImk0Emk2HKlCnYvHkzjhw5olne/+mnnwIAh2MkatKkSZg0aRJSUlJw9epVnDhxAgEBAcjIyIClpSUCAwNhbm4udJmVQv0ZGR0dDTMzMzg4OECpVAKQZkjjYrpnOHz5Guo3jvoi087Ozvj999+xfPlyHD16FPHx8ZKcA7Fq1aoSF2DPysqCvr6+5HqGAGDixImS70UFSi7vB1Dq8n4pkfpwTFnExcXhxo0beP/994UupdJt2LABKSkpGDt2rOYUIVLFaSDsKXst9S8WY2NjeHp6IjExERYWFgCeXSLE1tZWyPIEoVKp0KlTJ8ybNw/9+vVD27Zt4eHhoWkXqWAvqjYu79fG4Ziyq1evnuY0OlLTq1cvBAYGYuTIkWjRogXGjBkDT09PyfWWcRrIM+wp04FcLkdGRgZmzJiBmJgYuLu7Y/HixZI5M7m611B99m1PT0/o6enh0aNHyM/PR4sWLfDZZ58JXWalYS9qSVze/xxX5dLLFF88lpOTg+rVqyMrKwurVq1CdHQ0hg0bhn79+gldZqXhYrrn2FOmAyMjI9jb22PPnj24c+cOLC0tJdlTdu3aNQwfPhwjR45EYmIiMjIyEBMTo/klI5X5QuxFLcnIyAhGRkaoWbMmGjdujG7dumkt7wekc3z0798fXbp00QzHLFu2TGs4pnPnzkKXWGnUX7pPnjxBUlIS3NzchC5JUOoTCv/000/4888/YWBgAAsLCzg5OeHBgwc4deoU+vXrJ5n3ChfTPcdQ9grqN8SlS5ewefNmxMbGol27dujcuTO8vb1hbW0tqZNgqkOIXC7XnJetTp06qFOnDtzc3DSLAKTwIVLc6NGjIZfLUadOHezYsQMtW7aEu7s7PvroI6FLE4S6B1FPTw/VqlWDra2tVkCVyvHB4Zjnil/V4MqVKwgICIBcLoeRkREKCgpgYGAgmeMCeP4ecHBwQK1atdC7d2+0bt0axsbG+PbbbzXfKVIYwuQ0EG0cvnwF9fyP9957D5MmTYKJiQnOnDmDq1evIjY2FuvXr5fcRbczMzMxbNgwJCcno1evXmjfvj1at24tyV6hl5FyLyo9w+EYbceOHcPRo0cRFxeHIUOGwN/fX3Pb6tWrYWlpidGjRwtYoXASExOxefNmPH78GD169EDbtm0lFUI4DUQbe8peQX0OIWdnZ/Tv3x8A0LNnTwBAcnKy5CYuA8+G6lasWIF79+7hxo0bCAoKwqpVqyQ3X4i9qNq4vF8bh2O0+fr6Ij4+HhcuXEBAQADWrFmDli1b4r333sO5c+cwc+ZMANI4lY76Nd64cQN//vknfHx84Ovri8DAQMydOxfDhw/HvHnzhC6z0nAaiDaGspfYtWsX1q9fj169esHGxgaXLl2Cr6+v5napngDT2NgY7u7ucHd3R/fu3SU7X0jdwTx//nytXtRly5ZJshdV/cF66tQpyS/v53BMSTVr1sTEiRPh4OCAPn36IDQ0FMeOHcOmTZvwn//8B35+fgCkEeDVrzE1NRUPHz7EzZs34eDggNatW6NGjRqws7MDIL2rPXAayDMcvnyJ7OxsBAUF4dSpU7h27Rry8vLQpk0bzXUdpbLi8kVS+CWri8mTJ+Pnn3/W2qbuRZXiGcrj4+MRGBiI33//XbLL+zkcU5L6x1pOTk6JuXRS6lF+FfX1lekZqU4DYSh7id9++w1DhgzR/H3r1i0cO3YMZ86cQWRkJLZs2aL5dUfSUrwXNTc3F3379tXqRZUaLu8v3fbt2+Hl5YXExET8888/mD9/Pr755hvY2tpi2rRpQpdXadQh9fDhw7h48SIuXryIWrVqoXPnzmjfvj1cXV0l92NPLpfjzJkz+Pvvv2FhYQFnZ2cMHDhQUos/AE4DKQ1DWSmSk5Mxe/ZsGBsbo3///ujRo0eJbmQpHSicL6SNvaja1B+sK1euLLG8/8KFC2jZsiVWrFghmaHt4qR+bkM1uVyOAQMG4JtvvsHXX3+NXr16Yc+ePcjMzMSff/4JZ2dnoUusFOr3wL59+3Dy5EnUq1cPderUwdmzZ1GjRg18++23MDExEbrMSsPFdCUxlL3EvXv38Ntvv+HkyZPw9PRE37590b59e0mGEDVeDuQZ9qKW7rfffsORI0e0lvfXrFkTenp6MDQ0lNQPmdJIcThG/YPu3LlzOHToED7//HNMmzYNe/bswZ49e1BYWIhRo0YJXWalUbfHlClTMG7cOLRu3RrAs1XtX331FUaMGCHJzw5OA3lOugnjNerXr4/Zs2djx44dqFOnDnbt2oWff/4ZDx8+FLo0wfTq1QumpqYYOXIkZs2ahYiICE2PmVQkJyfjjz/+wKRJk3D06FEUFRXB3d0dn376KYKDgxEZGYk2bdoIXaYghgwZgsWLFyMqKgrLly9HWFgYsrKyNPNkpBDIFAoFAODSpUuYNGkSunfvjm+++QYnT56ElZUVbG1tIaXfwer/51FRUfD19cW///4LFxcXAM8m/0dERACAZD5H1D/qi4qKkJKSAuDZqIulpSUyMjI0C6akcIzs2rULHTt2xHfffadZTFecvb295AIZwJ6yEhQKBa5evYrk5GTk5eXB09MT+fn5CAwMxL59+2BhYYHg4GDJ9BRxvlBJ7EV9rrTl/QqFAoGBgTh79qzklvdzOKZ00dHRMDY2Rn5+PubOnQtXV1c8ePAA/fv3x6BBgyQ3tH3mzBls3LgRH3zwATw8PHDx4kUcOXIE+/btE7q0SsNpIKVjKHvB+fPnMWHCBM2pMG7fvo0GDRqgcePGiI2Nxc2bN7F//36hy6w0nC/0cqmpqdixYwciIyPRtGlTfPDBB5IJ6y86deoUDh48iMzMTDg4OKB+/fqIiopCw4YNMXHiRMkt7+dwzMudPn0aR48ehZubG4YOHSqZOVRPnz6FgYEBTE1NUVhYiP3792Pfvn3IyspC586d0aNHD/j6+kpm0QOngZSOoewF2dnZ2L9/P27fvo1OnTqhZ8+eUCgUKCgo0JzQToo4X4i9qLqQ4vJ+rsrVpv5MyM3NRWhoKAoLC+Hh4SHZHpDvvvsOFhYWcHd3R4MGDVC/fn0Az3qbFQqFZD5HAS6mexWGslLk5+fj0qVLCAwMRFZWFrp374733nsPVlZWQpcmKKlfDoS9qKXj8v5nOByjTR3Mly5disjISMTHxyMvLw+1atVCw4YNMWHCBMlcmFyhUGh61R88eIAnT57A09MTjRs3RpMmTWBnZ4f69etLoodMjdNASsdQVoxCoYBcLtfqTg8PD8e1a9fw5MkT+Pn5aVbLSAHnC2ljL6o2Lu/XxuGY0vXs2RO7du2CjY0NCgoKcP78eQQFBWH06NFo1aqVZIbr1M6cOYMVK1Zg4MCB+Oeff5CcnAwnJycsW7ZM6NIEwWkg2hjKirly5QpWrFiBJ0+ewMHBAY6Ojnj8+DHCw8ORkpICOzs7nDlzRugyKx3nCz3HXtTnuLz/OQ7HaHvw4AHu3r0LT09P/Prrr/D395fktYKLk8vlMDIywo8//oiaNWti7NixAJ6FkkePHsHT01MSAZXTQF6NoayYyMhIREZGwtzcHLGxsQCAWrVqoVq1akhISEDdunXx3nvvCVyl8KQ4X4i9qC/38ccfo3///ujTp48mePj7+2PmzJlo0aKFZMIIh2OeCwkJwbRp02BkZASZTAZfX198+umnmtNhSI1cLoehoSH09PQwevRofP755/Dy8pLkAilOA3k1hjJ6Jc4Xeoa9qC/H5f3aOBwDzTUuk5OTcfLkSRw9ehTh4eGoVq0a6tatiwULFsDb21voMitFQkICPvvsM3Tt2hWNGjXCqlWrcPDgQaHLEgyngbwaQ1kZSaFbuTjOF9LGXlRtXN7/HIdjtBUWFuKzzz5Dz5490bt3b63bIiMjERgYiCFDhsDFxUUSx0dWVhb27t2La9euITo6GpmZmejcuTM8PDzg4+OD+vXrw9zcXOgyKxWngbwcQxmVivOF6FW4vP85Dsdoy87OxpYtWxAUFAR9fX2MHj0a3bt3R506dYQuTRRu376NkJAQ/Pvvv7h69SqmTZuGCRMmCF1WpeA0kNeTxuxs0pkulwORwhfvq0jh135xCoUCderUQWRkJC5duiT55f1NmzbF3Llzcfv2bTRv3hxffPGFpIdjzM3NMX36dEyfPh179uzBP//8g/T0dAwaNEhypwUpTZMmTdCkSRMAz6aH5ObmApDG58jVq1dfOQ0kMDBQstNA1NhTRq/E+UL0Klze/wyHY56Ty+XIysrSnMPw/Pnz2LRpE27evImuXbvik08+gb29vcBVkhA4DeT1GMqoBM4Xotfh8v5nOByjLS0tDZ9++inq1q2L+Ph4GBgYaFag7t27F3FxcQgJCUHdunWFLpVIlBjKqATOF6JX4fL+57gqV9udO3cwc+ZMeHl5YezYsVCpVLh//z5MTU1hb2+PlJQUtGvXTugySYSk8COuLBjKSAsvB0KvwuX92jgcU1JMTIxmLlnnzp3Rr18/NGzYUHM7f9ARvRxDGb0U5wvRi7i8n8oqOTkZx48fR1ZWFry9vblam6gMGMqoBM4XorKS8vL+15HaeyQjIwMHDx6EjY0NHB0dkZubi9jYWFy9ehX//vsvrK2tsX37dhgbGwtdKpFo8ZQYpEU9Xwh4NmH5888/B/BsWNPGxgY2NjYAIKkvG3o5KS/vfx2pvf7bt28jNDQUqampSElJgYeHBywtLZGYmIjs7GzUr18fxsbGHL4kegWGMtJ4cb5QZmYmvLy8AEByE7hJd0ZGRprz10ktkBDg6+uLpk2bai7BlpSUBJVKpTlpbEZGBgDOKSN6FQ5fkgbnCxFReWMIIyo7hjJ6Kc4XIiIiqjwMZVQm6vlClpaWnC9ERERUARjKiIiIiESA3R1EREREIsBQRkRERCQCDGVEREREIsBQRkRERCQCDGVEREREIsBQRkSiN2rUKCxevLjE9gMHDqBly5aiqaeirVmzBh988EGl75eIKgdDGRHRSxQWFgpdAhFJCEMZEVUZly5dwuDBg9GsWTO0bNkSw4YNw8OHDzW3h4SEYMCAAfDy8kK3bt2wdu1aFBUVaW53dXXFr7/+ismTJ6NZs2b4+eefy7Tfrl274ueff8YXX3wBHx8fdO7cGXv37tXc/uDBA7i6uuLw4cMYNmwYvLy88P777yMsLExzn9J6/UJCQuDq6qq5fe3atYiMjISrqytcXV1x4MCBN2onIhInhjIiqhKKioowdepUtGrVCgcPHsTevXsxdOhQzXUXL1++jDlz5mD06NE4cuQIvv32Wxw4cKBE8Fq7di169OiBQ4cOYdCgQWXe/5YtW+Dp6Yng4GCMGDEC8+fPx927d7Xu8/3332PcuHEIDg5Gs2bNMHnyZM2Ful+nd+/eGD9+PBo3boxz587h3Llz6N27d5nrIyLxYygjoiohOzsbWVlZ6NKlC5ycnNCwYUMMGDAAderUAfAsbE2cOBEDBgyAo6Mj2rVrhxkzZmDPnj1az/P+++9j0KBBcHR01Dy2LDp27IiRI0fC2dkZH3/8MWrWrIlLly5p3WfkyJHo2bMnGjZsiPnz58PCwgL79+8v0/MbGxvD1NQUMpkMtra2sLW1hbGxcZnrIyLxMxC6ACKi8mBpaYmBAwdiwoQJaNeuHfz8/PDee+/Bzs4OABAZGYl///1Xq2dMoVCgoKAAeXl5MDExAQB4enq+0f7Vw4wAoKenBxsbG6SlpWndx8fHR/PfBgYG8PT0LNGbRkTSxVBGRKJnZmaG7OzsEtufPn0KCwsLzd8BAQEYNWoUzp49i6NHj2LlypXYsmULmjVrhtzcXEyfPh3/+c9/SjxPtWrVNP9tamr6RjUaGGh/nOrp6UGXSwvr6+uXuD8XGhBJC4cviUj06tevj5s3b5bYfuvWLdSrV09rm7u7OyZNmoQ9e/bAxcUFf/zxh2b7vXv34OzsXOKfvn7lfBRevXpV899FRUW4efMmGjRoAACoWbMmcnJykJubq7lPZGSk1uMNDQ2hVCorpVYiqnzsKSMi0RsxYgR27dqFRYsWYfDgwTAyMsLp06dx+PBh/PTTTwCAhIQE7Nu3D127doWdnR3u3buHuLg4zXm9pk6dismTJ6NOnTro2bMn9PX1ERkZiejoaMycObNSXsevv/6KevXqoUGDBti2bRuePHmiWUzQtGlTmJiYYMWKFRg9ejSuXbtWYnWlg4MDHjx4gNu3b8Pe3h7m5uYwMjKqlNqJqOIxlBGR6Dk6OmLnzp1YuXIlxo0bh8LCQjRo0ACrVq1Cx44dAQAmJia4e/cugoKCkJmZCTs7O4wcORLDhg0DAHTo0AE///wz1q1bh19++QUGBgZo0KABhgwZUmmvY9asWdiwYQNu374NZ2dn/PTTT7CysgLwbE7cDz/8gO+//x6//fYb/Pz8MH36dHz11Veax/fs2RN///03Ro8ejadPnyIgIAADBw6stPqJqGLpqXSZ9EBERDp78OABunXrhuDgYDRp0kTocohIpDinjIiIiEgEGMqIiIiIRIDDl0REREQiwJ4yIiIiIhFgKCMiIiISAYYyIiIiIhFgKCMiIiISAYYyIiIiIhFgKCMiIiISAYYyIiIiIhFgKCMiIiISAYYyIiIiIhH4/xPjsFN5oZlSAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_metric('context_recall')" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "81778279-2362-4cf8-aa2d-247e267a86d1", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAHMCAYAAACUdN+cAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABiEklEQVR4nO3deVhUZf8G8HsYQEAQZQ8EVFSQzV3EDUINRTOX3Jc0U7O0csnl91pqqWguqWlZmZpmrghmouWGK2KuKIILioIosir7wDC/P7xmZAQVEDlHzv25rvd648zMme88zpy551nOkalUKhWIiIiISFA6QhdARERERAxlRERERKLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRERERCLAUEZEREQkAgxlRFUoIiICzs7OiIiIKPN99+/fX6Z9h4SEoFu3bnBzc0OrVq1eW11ilZKSgs8++wxeXl5wdnbGhg0bhC6p0v3www9wdnZGWlqa0KUQ0WugK3QBRG+CXbt2YebMmaXeNmbMGEydOrXC+96zZw9SU1MxcuTICu8jNjYWM2fORMeOHTF27FgYGBhUeF9vqsDAQBw/fhwTJkyAhYUF3N3dX+vzOTs7P/e2gQMH4ptvvnmtz18Rw4cPR3p6Ov7+++8St6WlpcHb2xsTJkzAxIkTBahOW0REBEaMGIEVK1agW7duQpcDAFizZg0aNmyILl26CF0KVVMMZUTl8Nlnn6Fu3bpa2xo3blzmx7du3RqRkZHQ09PTbPv7779x48aNVwplZ86cQVFREf73v//B0dGxwvt5k50+fRqdO3fG6NGjq+w527dvj/fee6/E9vr161dZDVR1fv75Z/j7+zOU0WvDUEZUDp06dYKHh0eFH6+jo4MaNWpUYkVPpKamAgBMTEwqfd9vitTUVNSqVavS9pefnw89PT3o6Dx/lke9evVKDWVUNjk5OTAyMhK6DCLR4Jwyokpw7949zJkzB/7+/vD09ISXlxc+++wzJCQkaN3v2blbw4cPR1hYGO7duwdnZ2c4OzvDz89P6zFFRUX46aefNIHwgw8+wJ07dzS3+/n54YcffgAAeHt7w9nZWfN38f8uzs/PDzNmzHjhaxo+fDh69uyJmzdvYvjw4WjatCk6duyIX3/9tcR9FQoFVq5cia5du8Ld3R0+Pj747rvvoFAotO538uRJDB48GK1atULz5s3h7++PZcuWad1n06ZN6NGjB5o2bYrWrVujb9++2LNnz3Pr3LVrF5ydnaFSqbB582ZNO6rFx8fjs88+Q5s2bdC0aVMMGDAAYWFhWvtQ/7vs3bsX33//PTp27IimTZsiKyvrhW1UFmfPnsVnn30GX19fTdssWLAAeXl5Je4bGxuLzz//HG3btoWnpyf8/f3x/fffl7hfZmYmZsyYgVatWqFly5aYOXMmcnNzX7nWZ2VlZWH+/Pnw8/ODu7s7vL29MWrUKERFRWnd79KlSxg9ejRatmyJpk2bYtiwYTh37pzWfdTz4W7evIkpU6agdevWGDJkSLnqUe/jzp07L339zs7O+Oabb/DXX3/B398fHh4e6Nu3L/777z+t+82YMaPEZ674cxXfX05ODoKDgzXvsZd9hojKiz1lROWQlZVVYpK1mZkZLl++jAsXLqBHjx6wsbHBvXv3sGXLFowYMQJ79+6FoaFhqfv7+OOPkZmZiQcPHmjmrNWsWVPrPr/++itkMhk+/PBDZGVlYe3atZg6dSp27NgBAPi///s/hISE4MCBA5gzZw6MjIxeON+pPB49eoSPPvoIXbt2Rffu3fHPP/9gyZIlaNy4MXx8fAA8CY3jx4/HuXPnMGDAADg5OeH69ev4/fffERcXhx9//BEAcOPGDYwbNw7Ozs747LPPoK+vjzt37uD8+fOa59u+fTvmzZsHf39/jBgxAvn5+bh27RouXbqEd999t9QaW7duje+++w7Tpk0rMZyYkpKCQYMGITc3F8OHD0edOnUQHByM8ePHa0JkcT/++CP09PQwevRoKBQKrWHm0uTn55c66d7Y2Bj6+voAgP379yMvLw+DBw9G7dq1ERkZiT/++AMPHjzAypUrNY+JiYnB0KFDoauri4EDB8LOzg53797F4cOHMWnSJK39f/HFF6hbty4mT56Mq1evYseOHTAzM8OXX375wnrLa/bs2fjnn38wbNgwODk5ISMjA+fOnUNsbCzc3NwAAOHh4RgzZgzc3d0xYcIEyGQy7Nq1Cx988AH+/PNPeHp6au3z888/h6OjIyZNmgSVSlWhusr6+v/77z+EhoZi+PDh0NfXx5YtW/DRRx9hx44d5Zp2AADfffcdZs2aBU9PTwwYMAAA4ODgUKH6iZ6HoYyoHEqb93Xt2jX4+vqWmIz89ttvY+DAgfjnn3/Qu3fvUvfXvn17bNy4EY8fP37uMFh+fj5CQkI0X/K1atXC/Pnzcf36dTRu3BhdunRBdHQ0Dhw4AH9/f5iZmb3Sayzu4cOHWLRokab+999/H35+fggKCtKEsj179uDUqVPYtGmT1qrPRo0aYfbs2Th//jxatGiBkydPoqCgAL/++utzawwLC0OjRo20wsrL2Nvbw97eHtOmTSsxnPjLL78gJSUFmzdv1tTWv39/9OrVC4GBgejcubPW8GR+fj6CgoLKvFBi586d2LlzZ4nty5YtQ48ePQAAU6dO1drfwIED4ejoiGXLliExMRG2trYAgHnz5kGlUiE4OFizTf34ZzVp0gQLFizQ/J2RkYGdO3dWeig7evQoBgwYoNUjNGbMGM1/q1QqzJkzB15eXli7di1kMhkAYNCgQejRoweWL1+OdevWae3TxcUFS5cufaW6yvr6r1+/jqCgIM2ijx49eqBbt25YuXIlVq1aVa7nfO+99zBnzhzY29tzyJpeG4YyonL4+uuvS53EXfxLt6CgAFlZWXBwcECtWrVw9erV54aysujbt68mkAHQhIv4+Phy/9ovLyMjI60vIH19fXh4eCA+Pl6zbf/+/XByckKDBg20eo3atm0L4MnQYIsWLTTzvQ4dOoR+/fqVOlerVq1aePDgASIjI0v0sFTE0aNH4enpqRUWa9asiYEDB2Lp0qW4efOmVhv27t27XCtXO3fujGHDhpXYXnyfxfeXk5ODvLw8NG/eHCqVClevXoWtrS3S0tLw33//YcSIEVqBDIAm6BQ3aNAgrb9btWqFAwcOICsrC8bGxmWu/2Vq1aqFS5cuISkpCdbW1iVuj46ORlxcHMaPH4/09HSt27y9vbF7924UFRVp/Vs/W3tFlPX1N2/eXGsVrq2tLTp37owjR45AqVRCLpe/ci1ElYmhjKgcPD09S53on5eXh59//hm7du1CUlKS1rBMZmbmKz3ns1/S6nDz+PHjV9pvWdjY2JQIBaamprh27Zrm7zt37iA2Nhbe3t6l7kO9CCEgIAA7duzArFmzsHTpUnh7e6Nr167o1q2b5kt7zJgxOHXqFPr37w9HR0e0b98ePXv2RMuWLStUf2JiIpo2bVpie4MGDTS3Fw9Qz66sfRkbGxu0a9fupTWsXLkShw8fxqNHj7RuU89ZU4fcsobs570nHj169MqhrPi/99SpUzFjxgz4+vrCzc0NPj4+6N27N+zt7QEAcXFxAIDp06c/d3+ZmZkwNTXV/F3eNi5NWV9/aSuR69Wrh9zcXKSlpcHS0vKVayGqTAxlRJXg22+/1cyjadasGUxMTCCTyV5p3oza81b/vcp+lUplme5Xlp6EoqIiNG7c+LnncbOxsQHwpMdo8+bNiIiIQFhYGI4fP47Q0FBs27YN69atg1wuh5OTE/bv36+5/d9//8Wff/6JTz/9FJ999lnZX2AFVfb53ZRKJUaNGqWZm9egQQMYGRkhKSkJM2bMQFFRUYX2W9H3hL6+fqkLDABothfvlQ0ICND0Qp08eRK//fYbfv31V/zwww/w8fHRPN+0adPQpEmTUvf77OrKylh9XJmfidJ6IoGyf0aIKhNDGVElUM8bKz73Jj8/v0y9ZM/7UqgMpqamJXrUFAoFkpOTK+05HBwcEBMTA29v75e+Fh0dHXh7e8Pb2xszZ87EmjVr8P333yMiIkLT42RkZISAgAAEBARAoVBg4sSJWLNmDcaNG1fuL3RbW1vcvn27xPZbt25pbn+drl+/jri4OK15ecCTVajFqXuerl+//lrrsbOzQ0REBPLy8koEUHU72dnZaW23srLC0KFDMXToUKSmpqJPnz5Ys2YNfHx8NHUbGxu/tMdQCMVXKavFxcXB0NBQM6+xVq1apfY6JyYmvvb6iJ7FU2IQVYLSepQ2bdpUpl/bhoaGrzzE+Tz29vY4e/as1rbt27dXai9A9+7dkZSUhO3bt5e4LS8vDzk5OQCeTMZ+lrp3RX3qjGfnJenr68PJyQkqlQoFBQXlrs3HxweRkZG4cOGCZltOTg62b98OOzs7NGzYsNz7LA91j07xHhyVSoWNGzdq3c/MzAytW7dGUFBQiTDwqj2txXXq1AkFBQXYunWr1vaioiJs2bIFenp6mmFopVJZ4n1pbm4OKysrzb+Xu7s7HBwcsG7dOmRnZ5d4PqEvB3XhwgWt03fcv38fhw4dQvv27TWfWQcHB2RmZiImJkZzv4cPH+LAgQMl9mdkZFQl0wZIuthTRlQJfH19sXv3bhgbG6Nhw4a4ePEiTp06hdq1a7/0sW5ubggNDUVgYCA8PDxgZGRU6nmTKqJ///6YPXs2Jk6ciHbt2iEmJgYnTpxAnTp1KmX/wJNVafv27cPs2bM1k/qVSiVu3bqF/fv3Y+3atfDw8MDq1atx9uxZ+Pj4wM7ODqmpqfjzzz9hY2OjmTM2evRoWFhYoEWLFjA3N8etW7fwxx9/wMfHp0JzpcaOHYu9e/dizJgxGD58OExNTRESEoKEhAT88MMPLzwxbFnExcVh9+7dJbZbWFigffv2aNCgARwcHLBo0SIkJSXB2NgY//zzT6lf7LNmzcLgwYPRp08fDBw4EHXr1sW9e/cQFhZW6nNUhJ+fHzp06IDAwEBcvnwZzZs3R25uLg4fPozz58/jiy++0PQgZWdnw8fHB/7+/nBxcYGRkRFOnTqFy5cva3qEdXR0MG/ePIwZMwY9e/ZE3759YW1tjaSkJERERMDY2Bhr1qyplNoronHjxhg9erTWKTEAaF1GKiAgAEuWLMGECRMwfPhw5OXlYcuWLahfv36J87G5ubkhPDwc69evh5WVFerWrVvqnEWiimIoI6oE//vf/6Cjo4M9e/YgPz8fLVq0wPr16/HRRx+99LFDhgxBdHQ0du3ahQ0bNsDOzq7SQtmAAQOQkJCAnTt34vjx42jZsiXWr1//Spd0epaOjg5Wr16NDRs2YPfu3Thw4AAMDQ1Rt25dDB8+XLNa1c/PD/fu3UNQUBDS09NRp04dtGnTBhMnTtRciWDgwIHYs2cP1q9fj5ycHNjY2GD48OH45JNPKlSbhYUFtm7disWLF+OPP/5Afn4+nJ2dsWbNGvj6+r7yaz958mSJoUgAaNOmDdq3bw89PT2sWbMG8+bNw88//4waNWqga9euGDp0aInTKri4uGD79u1YsWIFtmzZgvz8fNja2qJ79+6vXKeajo4OfvrpJ/zyyy/Yu3cv/v33X+jq6qJx48ZYvHgxevXqpbmvgYEBBg8ejJMnT+Lff/+FSqWCg4MDZs+erXXSVy8vL2zbtg0//vgj/vjjD+Tk5MDS0hKenp4YOHBgpdVeEa1bt0azZs2wevVqJCYmomHDhggMDISLi4vmPnXq1MGqVauwcOFCLF68WHP+szt37pQIZTNmzMDXX3+N5cuXIy8vD3369GEoo0olU1Vm3zgREZEIODs7Y+jQofj666+FLoWozDinjIiIiEgEGMqIiIiIRIChjIiIiEgEOKeMiIiISATYU0ZEREQkAgxlRERERCIgufOUFRUVobCwEDo6Oq/18jZEREREwJMrcxQVFUFXV/eFJ62WXCgrLCzE5cuXhS6DiIiIJMbDwwP6+vrPvV1yoUydUD08PEq9XiERERFRZVIqlbh8+fJLL+0muVCmHrKUy+UMZURERFRlXjZtihP9iYiIiESAoYyIiIhIBBjKiIiIiESAoYyIiIhIBBjKiIiIiESAoYyIiIhIBBjKiIiIiESAoYyIiIhIBBjKiIiIiESAoYyIiIhIBBjKiIiIiESAoYyIiIhIBBjKiIiIiESAoYyIiIhIBBjKiIiIiESAoYyIiIhIBBjKiIiIiESAoYyIiIhIBBjKiIiIiESAoYyIiIhIBBjKiIiIiERA0FD233//4eOPP0aHDh3g7OyMgwcPvvQxERER6NOnD9zd3dG1a1fs2rWrCiolIiIier0EDWU5OTlwdnbG7Nmzy3T/+Ph4jBs3Dl5eXti9ezc++OADzJo1C8ePH3/NlRIRERG9XrpCPrmPjw98fHzKfP+tW7eibt26mDFjBgDAyckJ586dw4YNG9CxY8fXVSYRERHRaydoKCuvixcvwtvbW2tbhw4dsGDBgnLvS6lUVlZZFfLTTz9h9+7deO+99zB+/HhBayEiIqLXp6yZ440KZSkpKbCwsNDaZmFhgaysLOTl5cHAwKDM+7p8+XJll1dmCoUCISEhUKlU2L17N1q2bAl9fX3B6nkePT09uLq6Qlf3jXqblElhYSGuXr2KgoICoUt5Iz15b7hBV1cudCmvRWGhElevRpX5/aGnpwdXN1foyqvfZwUACpWFuBpV9s9LdW6P8rYFwPZ4lp6eHtzcXCGvhu2hVBYiqpztUVz1a5Ey8vDwgFwuzBdKVlYWVCoVAKCoqAiurq4wNjYWpJaXkcvl+O77XbibkCx0KZXGoa4lpk3qCzc3N6FLeaPJ5XLM2rYLtx+mCF1KpapvZYF5A8v//pDL5Zi5bydupVWv9mhgZoHA7u9XqD0Whm/D3ccPX1NlVc+hlhVmeA+s0LFDLpdj69V1eJhz/zVUJgwro7cwyPXDCrfH6Rvz8Tj37muoTBi1DB3QttH/Sm0PpVJZps6gNyqUWVhYICVF+4CXkpICY2PjcvWSAU/eEEKFsmefV8hayuJuQjJibz0QuoxKJ+Y2f1PcfpiCa4nV770BVOz9cSstBTEPq8+XbnEVaY+7jx/iZnria6hGWBU9djzMuY/ErPhKrkZ4FW2Px7l3kZFzo5KrEd6rfLe8Uecpa9asGU6fPq217dSpU2jWrJkwBRERERFVEkFDWXZ2NqKjoxEdHQ0ASEhIQHR0NBITn/yyWrp0KaZNm6a5/6BBgxAfH4/vvvsOsbGx2Lx5M/bt24eRI0cKUT4RERFRpRF0+PLKlSsYMWKE5u/AwEAAQJ8+fbBw4UIkJyfj/v2nQwH29vb4+eefERgYiI0bN8LGxgbz5s3j6TCIiIjojSdoKPPy8sK1a9eee/vChQtLfUxISMhrrIqIiIio6r1Rc8qIiIiIqiuGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCgjIiIiEgGGMiIiIiIRYCh7RpGySOgSXovq+rqIiIiqC12hCxAbHbkOFs/aiPjbSa/tOZRFBVp/Tx+zEnIdvdf2fPb1rfHlvBGvbf9ERET06hjKShF/OwmxMQmvbf8qmRIwffr37RuJkKnkr+35iIiISPw4fElEREQkAgxlRERERCIgeCjbvHkz/Pz84OHhgf79+yMyMvKF99+wYQP8/f3h6ekJHx8fLFiwAPn5+VVULREREdHrIWgoCw0NRWBgID799FMEBwfDxcUFo0ePRmpqaqn337NnD5YuXYoJEyYgNDQU8+fPR2hoKJYtW1bFlRMRERFVLkFD2fr16zFgwAD069cPDRs2xNy5c2FgYICgoKBS73/hwgW0aNEC7777LurWrYsOHTqgZ8+eL+1dIyIiIhI7wVZfKhQKREVFYdy4cZptOjo6aNeuHS5cuFDqY5o3b46//voLkZGR8PT0RHx8PI4ePYr33nuv3M+vVCpL3S6XV99VkM97zS/C9qDnqc7vDaD87w+2h7bq3B48lmpje2grrT3K2kaChbL09HQolUqYm5trbTc3N8etW7dKfcy7776L9PR0DBkyBCqVCoWFhRg0aBA+/vjjcj//5cuXS2wzNDSEq6truff1prh27Rpyc3PLfH+2hzY9PT24urpBV7f6HUwKC5W4ejUKBQUFL78zqv97Ayjf+4Ptoa26twePpdrYHtrK2x7FvVHnKYuIiMDPP/+M2bNnw9PTE3fv3sX8+fOxevVqfPrpp+Xal4eHR7VO6qVxdnYWugRRqUh7yOVyfPNbMO7cT3kNFQnD8S0LfD26D9zc3IQuRVT4edHG9niKbaGN7aGttPZQKpWldgY9S7BQVqdOHcjl8hKT+lNTU2FhYVHqY1asWIFevXqhf//+AJ688JycHHz99dcYP348dHTKPkVOLpdLLpRJ7fW+TEXb4879FFyPf1DJ1QiP7w9tbA9tbI+n2Bba2B7aXqU9BJvor6+vDzc3N4SHh2u2FRUVITw8HM2bNy/1MXl5eSWCl/rFq1Sq11csERER0Wsm6PDlqFGjMH36dLi7u8PT0xO///47cnNz0bdvXwDAtGnTYG1tjSlTpgAA3n77baxfvx6urq6a4csVK1bg7bffZlInIiKiN5qgoSwgIABpaWlYuXIlkpOT0aRJE6xdu1YzfHn//n2tnrHx48dDJpNh+fLlSEpKgpmZGd5++21MmjRJqJdAREREVCkEn+g/bNgwDBs2rNTbNm3apPW3rq4uJkyYgAkTJlRFaURERERVRvDLLBERERERQxkRERGRKDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCFQolAUHByMsLEzz93fffYdWrVph0KBBuHfvXmXVRkRERCQZFQpla9asQY0aNQAAFy5cwJ9//okvv/wStWvXRmBgYKUWSERERCQFuhV50IMHD+Do6AgAOHjwIN555x0MHDgQLVq0wPDhwyu1QCIiIiIpqFBPmZGRETIyMgAAJ0+eRLt27QAANWrUQH5+fqUVR0RERCQVFeopa9euHWbNmoUmTZogLi4OPj4+AIAbN27Azs6uUgusllQyQAVAhif/r5IJXBAREREJrUI9ZbNnz0azZs2QlpaGlStXok6dOgCAqKgo9OjRo1ILrI5k0IFcYQ6oALnCHDIugiUiIpK8CvWU1apVC19//XWJ7Z999tkrFyQVerlvQS/3LaHLICIiIpGoUBfNsWPHcPbsWc3fmzdvxnvvvYcpU6bg0aNHlVYcERERkVRUKJQtXrwY2dnZAIBr165h4cKF8PHxQUJCAhYuXFipBRIRERFJQYWGLxMSEuDk5AQA+Pfff/H2229j8uTJiIqKwtixYyu1QCIiIiIpqFBPmZ6eHvLy8gAAp06dQvv27QEApqamyMrKqrzqiIiIiCSiQj1lLVq0QGBgIFq0aIHLly9j+fLlAIC4uDjY2NhUZn1EREREklChnrKvv/4aurq6+OeffzB79mxYW1sDeLIAoGPHjpVaIBEREZEUVKinzNbWFj///HOJ7f/3f//3ygURERERSVGFz1p69+5dfP/995g8eTJSU1MBAEePHsWNGzcqrTgiIiIiqahQKDtz5gzeffddREZG4t9//0VOTg6AJ6fH+OGHHyq1QCIiIiIpqFAoW7p0Kb744gusX78eenp6mu1t27bFxYsXK6s2IiIiIsmoUCi7fv06unTpUmK7mZkZ0tPTX7koIiIiIqmpUCgzMTFBcnJyie3R0dGalZhEREREVHYVCmU9evTAkiVLkJycDJlMhqKiIpw7dw6LFi1C7969y7WvzZs3w8/PDx4eHujfvz8iIyNfeP/Hjx9j7ty56NChA9zd3eHv74+jR49W5GUQERERiUaFTokxadIkfPPNN/D19YVSqUSPHj2gVCrRs2dPjB8/vsz7CQ0NRWBgIObOnYumTZvi999/x+jRo7F//36Ym5uXuL9CocCoUaNgbm6OFStWwNraGomJiahVq1ZFXgYRERGRaFQolOnr62PevHn45JNPcOPGDWRnZ8PV1RX16tUr137Wr1+PAQMGoF+/fgCAuXPnIiwsDEFBQaVeQzMoKAiPHj3C1q1bNQsM6tatW5GXQERERCQqFQplara2trC1ta3QYxUKBaKiojBu3DjNNh0dHbRr1w4XLlwo9TGHDx9Gs2bN8M033+DQoUMwMzNDz549MWbMGMjl8nI9v1KpLHV7effzJnnea34Rtoc2tsdT1bktALbHs9geT/HYoY3toa209ihrG1UolCmVSuzatQunT59GamoqioqKtG7fuHHjS/eRnp4OpVJZYpjS3Nwct27dKvUx8fHxOH36NN5991388ssvuHv3LubOnYvCwkJMmDChXK/h8uXLJbYZGhrC1dW1XPt5k1y7dg25ubllvj/bQxvb46nq3hYA2+NZbI+neOzQxvbQVt72KK5CoWz+/PkIDg6Gj48PGjVqBJlMVqEnLy+VSgVzc3N8++23kMvlcHd3R1JSEn777bdyhzIPD49qndRL4+zsLHQJosL20Mb20Mb20Mb2eIptoY3toa209lAqlaV2Bj2rQqFs7969WL58OXx8fCrycABAnTp1IJfLNZdoUktNTYWFhUWpj7G0tISurq5WmGrQoAGSk5OhUCigr69f5ueXy+WSC2VSe70vw/bQxvbQxvbQxvZ4im2hje2h7VXao0KnxNDT04ODg0OFnxR4sljAzc0N4eHhmm1FRUUIDw9H8+bNS31MixYtcPfuXa3h0ri4OFhaWpYrkBERERGJTYVC2YcffoiNGzdCpVK90pOPGjUK27dvR3BwMGJjYzFnzhzk5uaib9++AIBp06Zh6dKlmvsPHjwYGRkZmD9/Pm7fvo2wsDD8/PPPGDp06CvVQURERCS0Cg1fnjt3DhERETh27BgaNWoEXV3t3axatapM+wkICEBaWhpWrlyJ5ORkNGnSBGvXrtUMX96/fx86Ok9z41tvvYXffvsNgYGB6NWrF6ytrTFixAiMGTOmIi+DiIiISDQqFMpq1aqFrl27VkoBw4YNw7Bhw0q9bdOmTSW2NW/eHNu3b6+U5yYiIiISiwqFssDAwMqug4iIiEjSKjSnjIiIiIgqV5l7ynr37l3m85EFBwdXuCAiIiIiKSpzKOvSpcvrrIOIiIhI0socysp7xnwiIiIiKjvOKSMiIiISgQqtvnRxcXnh/LLo6OgKF0REREQkRRUKZc+eHLawsBDR0dEIDg7GxIkTK6UwIiIiIimpUCgrbdJ/t27d0LBhQ4SGhqJ///6vXBgRERGRlFTqnLJmzZrh9OnTlblLIiIiIkmotFCWl5eHjRs3wsrKqrJ2SURERCQZFRq+bN26tdZEf5VKhezsbBgYGGDx4sWVVhwRERGRVFQolP3f//2f1t8ymQxmZmZo2rQpTE1NK6UwIiIiIikp18ljFy5cCGNjY8hkMgQEBEBfX/911kZEREQkGWWeUxYWFoacnBwAwMyZM5GZmfnaiiIiIiKSmjL3lNWvXx/Lli2Dl5cXVCoV9u3bB2Nj41Lv27t378qqj4iIiEgSyhzK5s6di4ULFyIsLAwymQzLly8v9az+MpmMoYyIiIionMocylq0aIHt27cDeHKZpX/++Qfm5uavrTAiIiIiKanQecoOHToEMzOzyq6FiIiISLIqdEoMOzs7PH78GJGRkUhNTYVKpdK6ncOXREREROVToVB2+PBhTJ06FTk5OZpTZKhxThkRERFR+VUolC1atAj9+vXD5MmTYWhoWNk1EREREUlOheaUJSUlYcSIEQxkRERERJWkQqGsQ4cOuHz5cmXXQkRERCRZZR6+PHTokOa/fXx8sHjxYsTGxqJx48bQ1dXeTefOnSuvQiIiIiIJKHMo+/TTT0tsW716dYltMpkM0dHRr1YVERERkcSUOZTFxMS8zjqIiIiIJK1Cc8pCQkKgUChKbFcoFAgJCXnVmoiIiIgkp0KhbObMmcjMzCyxPTs7GzNnznzlooiIiIikpkKhTKVSlXox8qSkJJiYmLxyUURERERSU66Tx/bu3RsymQwymQwffPCB1qpLpVKJhIQEdOzYsdKLJCIiIqruyhXKunTpAgCIjo5Ghw4dULNmTc1tenp6sLOzwzvvvFO5FRIRERFJQLlC2YQJEwA8uSB5QEAAatSo8VqKIiIiIpKaCl37sk+fPpVdBxEREZGklTmUtWnTBvv374eZmRlat25d6kR/tTNnzlRKcURERERSUeZQNnPmTBgbGwMA/u///u+1FUREREQkRWUOZcWHLDl8SURERFS5KjSnrLj8/HwUFBRobVP3qBERERFR2VQolOXk5GDJkiXYt28fMjIyStzOC5ITERERlU+Fzui/ePFinD59GnPmzIG+vj7mzZuHiRMnwsrKCosWLarsGomIiIiqvQqFsiNHjmD27Nnw9/eHXC5Hq1at8Mknn2DSpEnYs2dPZddIREREVO1VKJQ9evQI9vb2AJ7MH3v06BEAoGXLljh79mzlVUdEREQkERUKZXXr1kVCQgIAoEGDBti3bx+AJz1ovCA5ERERUfmVK5TFx8ejqKgI/fr1Q0xMDABg7Nix2Lx5Mzw8PBAYGIjRo0e/lkKJiIiIqrNyrb585513cOLECYwcORIA8MUXX2DWrFnYt28foqKi4ODgABcXl9dRJxEREVG1Vq6eMpVKpfX30aNHkZubCzs7O7zzzjsVDmSbN2+Gn58fPDw80L9/f0RGRpbpcXv37oWzszM++eSTCj0vERERkVhUaE5ZZQoNDUVgYCA+/fRTBAcHw8XFBaNHj0ZqauoLH5eQkIBFixahVatWVVQpERER0etTrlAmk8leeCHyili/fj0GDBiAfv36oWHDhpg7dy4MDAwQFBT03McolUpMnToVEydO1KwCJSIiInqTlWtOmUqlwowZM6Cvrw8AUCgUmDNnDgwNDbXut2rVqjLtT6FQICoqCuPGjdNs09HRQbt27XDhwoXnPm716tUwNzdH//79ce7cufK8BA2lUlnqdrlcXqH9vQme95pfhO2hje3xVHVuC4Dt8Sy2x1M8dmhje2grrT3K2kblCmXPXoi8V69e5Xl4Cenp6VAqlTA3N9fabm5ujlu3bpX6mLNnz2Lnzp0ICQl5pee+fPlyiW2GhoZwdXV9pf2K2bVr15Cbm1vm+7M9tLE9nqrubQGwPZ7F9niKxw5tbA9t5W2P4soVygIDAyv0JJUlKysL06ZNw7fffgszM7NX2peHh0e1TuqlcXZ2FroEUWF7aGN7aGN7aGN7PMW20Mb20FZaeyiVylI7g55VoQuSV5Y6depALpeXmNSfmpoKCwuLEvePj4/HvXv3MH78eM22oqIiAICrqyv2798PBweHMj23XC6XXCiT2ut9GbaHNraHNraHNrbHU2wLbWwPba/SHoKGMn19fbi5uSE8PBxdunQB8CRkhYeHY9iwYSXu36BBgxLX1ly+fDmys7Pxv//9DzY2NlVSNxEREVFlEzSUAcCoUaMwffp0uLu7w9PTE7///jtyc3PRt29fAMC0adNgbW2NKVOmoEaNGmjcuLHW42vVqgUAJbYTERERvUkED2UBAQFIS0vDypUrkZycjCZNmmDt2rWa4cv79+9DR0fw06kRERERvVaChzIAGDZsWKnDlQCwadOmFz524cKFr6MkIiIioirFLigiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhjIiIiIiERBFKNu8eTP8/Pzg4eGB/v37IzIy8rn33b59O4YMGYLWrVujdevWGDly5AvvT0RERPQmEDyUhYaGIjAwEJ9++imCg4Ph4uKC0aNHIzU1tdT7R0REoEePHti4cSO2bt2Kt956Cx9++CGSkpKquHIiIiKiyiN4KFu/fj0GDBiAfv36oWHDhpg7dy4MDAwQFBRU6v2XLl2KoUOHokmTJnBycsK8efNQVFSE8PDwKq6ciIiIqPLoCvnkCoUCUVFRGDdunGabjo4O2rVrhwsXLpRpH7m5uSgsLISpqWm5nlupVJa6XS6Xl2s/b5LnveYXYXtoY3s8VZ3bAmB7PIvt8RSPHdrYHtpKa4+ytpGgoSw9PR1KpRLm5uZa283NzXHr1q0y7WPJkiWwsrJCu3btyvXcly9fLrHN0NAQrq6u5drPm+TatWvIzc0t8/3ZHtrYHk9V97YA2B7PYns8xWOHNraHtvK2R3GChrJX9csvvyA0NBQbN25EjRo1yvVYDw+Pap3US+Ps7Cx0CaLC9tDG9tDG9tDG9niKbaGN7aGttPZQKpWldgY9S9BQVqdOHcjl8hKT+lNTU2FhYfHCx/7222/45ZdfsH79eri4uJT7ueVyueRCmdRe78uwPbSxPbSxPbSxPZ5iW2hje2h7lfYQdKK/vr4+3NzctCbpqyftN2/e/LmP+/XXX/Hjjz9i7dq18PDwqIpSiYiIiF4rwYcvR40ahenTp8Pd3R2enp74/fffkZubi759+wIApk2bBmtra0yZMgXAkyHLlStXYunSpbCzs0NycjIAwMjICDVr1hTsdRARERG9CsFDWUBAANLS0rBy5UokJyejSZMmWLt2rWb48v79+9DRedqht3XrVhQUFOCzzz7T2s+ECRMwceLEKq2diIiIqLIIHsoAYNiwYRg2bFipt23atEnr78OHD1dFSURERERVSvCTxxIRERERQxkRERGRKDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYkAQxkRERGRCDCUEREREYmAKELZ5s2b4efnBw8PD/Tv3x+RkZEvvP++ffvQrVs3eHh44N1338XRo0erqFIiIiKi10PwUBYaGorAwEB8+umnCA4OhouLC0aPHo3U1NRS73/+/HlMmTIF77//PkJCQtC5c2d8+umnuH79ehVXTkRERFR5BA9l69evx4ABA9CvXz80bNgQc+fOhYGBAYKCgkq9/8aNG9GxY0d89NFHcHJywhdffAFXV1f88ccfVVw5ERERUeXRFfLJFQoFoqKiMG7cOM02HR0dtGvXDhcuXCj1MRcvXsTIkSO1tnXo0AEHDx4s03OqVCrNc8vl8hK3y+Vy1Gv0FvT0S972prJztIJSqYRSqSz3Y+VyOeo7WkFPV/D8Xmnq2lm8Uns42VlWq/ZwsDavUHvI5XI0sraEvrz6tAUAOFpUvD0am1tCX6d6tUe9OhVvj/q1rKEnqz7H0romr3bssDG0gxzVpz0sDW1eqT1qGdSHTKX3GioTholB3ee2h3qbOoM8j6ChLD09HUqlEubm5lrbzc3NcevWrVIfk5KSAgsLixL3T0lJKdNzFhUVAQCuXr363Pv4vNsEQJMy7e9NcfHixQo/1re9AwCHSqtFDF6lPfw97QHYV1otYlDR9njXyQFwql7vDaDi7dHbsh5gWa8ySxGFiraHn74TYO5UucUI7FWOHU7wgJPMo/KKEVreq7WHHrrCrPKqEV7my9tDnUGeR9BQJgRdXV14eHhAR0cHMplM6HKIiIiomlOpVCgqKoKu7otjl6ChrE6dOpDL5SUm9aemppboDVOzsLAo0Sv2ovs/S0dHB/r6+hUrmIiIiOg1EXTyg76+Ptzc3BAeHq7ZVlRUhPDwcDRv3rzUxzRr1gynT5/W2nbq1Ck0a9bsdZZKRERE9FoJPiN11KhR2L59O4KDgxEbG4s5c+YgNzcXffv2BQBMmzYNS5cu1dx/xIgROH78ONatW4fY2Fj88MMPuHLlCoYNGybUSyAiIiJ6ZYLPKQsICEBaWhpWrlyJ5ORkNGnSBGvXrtUMR96/fx86xVYztWjRAkuWLMHy5cuxbNky1KtXD6tXr0bjxo2FeglEREREr0ymetn6TCIiIiJ67QQfviQiIiIihjIiIiIiUWAoIyIiIhIBhjIiIiIiEWAoIyIiIhIBhrIqVFhYKHQJRERE1dqbfFIJhrIqoH6DhIaGIiMjQ9hiREKpVCIvL0/oMkQjPj4eu3fvFroMwak/K2lpabh27ZrA1QhP3R7Z2dkCVyIuOTk5QpdAIqO+0Hd+fj5kMtkbG8wYyqqATCZDQUEBrl69itq1awtdjqDUH5RLly7h0KFDAlcjHgkJCTAyMhK6DMHJZDIAQGRkJI4cOQLg6cFWitTtMWrUKKSlpQlcjXhs375d8j/q1MfS48eP49dffxW4GuGpPytTpkzBqVOnNH+/aXjy2CqgUqne2DfI63LixAmYmJigadOmkm+foqIi6OjoSL4dilMoFCgqKoKBgYHQpYhCSkqK5ionUpeZmYktW7Zg7NixQpciCrGxsXj48CG8vb2hVCohl8uFLklQaWlpMDU1fWPbgaHsNVN/4d66dQv29vbQ09MTuiQSmcLCQqSmpsLa2lroUkRD/bkhaL5o2Sb8gUsvVh0+I2929W8A9Rtk8+bNSElJEbgaYRWfLxQTEyNwNeJx6dIl/Pjjj0KXITj1++PatWvYsmWLwNUIT90ee/fuLXENYClSB7LTp08jNzdX6HJEITs7G+Hh4UKXITj1FIczZ87g+vXrAlfzaqT9Ka8iRUVF6N+/P9566y2hSxFU8flCYWFhAKQ9X0jN1dUVn3/+udBlCE79/lAqlZqhOim/P9TtcffuXZibmwtcjfDU7XHw4EHJB1S1qKgo3Lp1C8CbveLwVanfD+fPn3/j3xscvqQqJ+X5Qhx+obLie6WkwsJC3L9/H/b29kKXIgrZ2dnIy8uDubk53y/VxJsdKUVKnXOVSiUAYMuWLbh//76QJVW5F2V9XV1dSQYyACUOmunp6di5c6dA1YiPQqHA0aNHhS5DcEVFRZDJZEhOTgYg7V6Q4nR1dRnIiqlZs6amF1WqgUz92UhNTUVWVpbA1bw6hrLXQP3hUK/+uHv3LmrUqCFkSVVG/QEpPhRVVFSkGYaS8nyh8PBwxMfHa22rLgeSynLr1i3cvHkTgLSDiHoIZsmSJYiJiZHsF676h636vbBgwQI8fPhQyJIEpz6W3rp1C3PmzBG2GIGUdmwIDQ3FyZMnBaimcjGUVYLib5A7d+4gLCwMDx480GybPn06zMzMhCityslkMvzzzz+4c+cOgCfBVEdHR/Mlo1AoJDdfKDMzEwCwYcMGTQ+h+sumTp06GDlypFClCU6lUiEjI0MzcbtBgwbo3bu3sEWJyMSJE+Hi4iJ0GYJR/7CVyWQoKiqCnZ0drKysBK6q6vz333/Iy8tDVlaW5nipPpbWrFkTAQEBAKRzLFWTyWS4fv068vLyND9YOnbsiA4dOghc2avTFbqA6qCwsBB6enpYvnw5rl+/DpVKhaysLNSqVQs2NjYYOHAgGjduLHSZVUKpVGL79u24e/cujIyM4OXlhZYtW8LLywu1a9eGh4cHPDw8AOCNn5BZFnl5eThy5AjOnDmDGzduID4+HjKZTBNMv/32W3z77bcwMTERuNKqVVhYiO+++w7h4eFwd3dHgwYN0LBhQ9StWxeNGjUCIN3hmOLq1q0rdAlVSn1Kg7S0NOzYsQOXLl3Cxx9/DE9PT+jo6OCDDz6oFqc9KKuwsDC0bt0aK1euRM2aNeHo6Ii6devirbfegrW1teY0OlJoD/W/e2xsLA4dOoSjR4/CyckJ33zzDe7fvw9bW1vo6+sLXeYrYyirBOpzjx06dAjTp0+HnZ0dUlNTcfv2bZw7d05S17yUy+X47bffEBMTg99++w1Hjx5FZGQkFi1aBDs7O7Ro0QKTJk0SuswqU1RUBCsrK9y/fx9KpRIhISEoLCyEtbU1cnNzERcXJ6lApj6whoWFITo6Gl999RUuX76Mq1ev4siRI7CxscGyZcuELlMUpDxxe+bMmejcuTPOnTuH9PR0AE96mt9++204OjoKXF3V+fDDD1FQUIDc3Fzcv38fly9fhqGhIczNzWFnZ4dBgwZJ5j2iHpHatm0bLCws4ODgoOk1PXbsGGrWrImePXsKWWKlYCh7RUlJSbhz5w7Mzc3h6uqq6T6tX78+PD094efnJ6nl7OoTXR46dAh2dnaYNGkSzMzM8O+//2LDhg1aiyDe1DMul4eRkRHatm0LU1NTpKWlQVdXF9HR0UhKSoKuri4++eQToUusUupf9ElJSejWrRvatGmDNm3aAAAyMjI0w/5S6g1RKBQ4ePAgunXrpvWapfJlW5yOjg7y8/ORmpqKAQMGYNeuXWjatCkA4K+//oKfn5/AFVYt9XfH9OnTUVBQgNjYWMTExODy5ct4+PCh5hqPUnivqD8bkZGR2Lp1K6ZMmYKWLVsCAI4ePQp/f38hy6s0DGWv6MqVK/jtt99gYWGB27dvY8GCBejTpw8aNWoEfX19SQUy4OkXyZkzZzBlyhTY2toCAHr16oWrV6/Cy8sLgDS624tr0qSJ5r+9vLyQnZ0NPT29atHdXhGdOnXChg0bcPLkSTg5OcHMzAy1a9dG7dq1oVKpJPH+yMvLg4GBAS5fvoywsDAEBARovmCzs7ORkZEBOzs7ocusclFRUXBxcUF6ejoMDAxQu3ZtJCYmAgAcHBwErq7qqX/I6unpwcXFBS4uLujduzcUCoXAlVUt9efC3t4e586dQ2xsLLy9vQEA9+7dqxbzyQCGslfm4eGByZMn4/79+7hx4wauX7+OlStXQk9PD8bGxvjggw/g7OwsdJlVRn0NxxYtWmDVqlUYP348HB0dYWZmhhMnTuD9998XukTBFJ+oW7NmTYGrEU5GRga+/fZbXLlyBQ8fPoSdnR0cHBxgY2MDHx8fSfSgAk+uWbh7926cOnUKjRs3RnJyMkxMTGBgYIAzZ87gwoULmDx5stBlVrnGjRvD0NAQQ4cOhYeHBxQKBf766y/NvFyp9LKrFf+RolKpNH+rf9BJoZcMePLaa9asiT59+uDnn3+GiYkJIiIisHv3btSrV6/adIAwlL0iKysrWFlZ4ejRoxg7dixSU1ORmJiIe/fu4dq1a5L88pXJZBg5ciR++OEHBAcHo7CwEFFRUWjRogUaNmyouY/USKH350XUQ5KHDh2CgYEBDhw4gPDwcFy9ehWnTp2Cubm5pIanLCws4OXlhe3bt8PW1hZz5syBXC6Hvb09Tp48idGjRwtdoiCMjY0xePBg5OXl4fHjx+jWrRv8/Pzw8ccfA5DesUN93CgqKpLMUGVp1K/b3d0dPj4+2Lt3L9atW4e2bdvinXfeEbi6ysMz+r8ChUIBXV1dhIeHY/369Vi7dq3mtvz8fERHR6NZs2bCFSiwpKQkhIeHQ6lUolGjRnByckLNmjUleWDJz89HSkoKrK2toasrzd9C6lC2Zs0a2NjYlDj1RWpqKszNzSU1n6yoqAiRkZEwNzfH3bt3ER8frzmv4ejRo2FsbCx0iVWusLAQurq6iImJQWpqKlxcXGBsbCyZcz0CT3sDjxw5ApVKBW9vbxgaGmrdR4rH0Wc9fvwYtWrVErqMSiXNb4dKcuvWLZw8eRLHjx+HlZUVMjIykJ+fD2tra5w+fRqXLl2SdCiztrYu9ZxTUjuQbN26Ffv27UPNmjWxZMkSzZeO1MKZOmilpKTg33//RWFhIVxdXWFjYwMzMzPNpWKkEMjUX7pnz55FamoqmjVrJukz1auDeHx8PP7880/s3bsX3t7ecHd3R9OmTSUVyICn52c7deoUtm7dqhmR6dSpEzp27Ah3d3fJHUfV1Ccj19XVrXaBDODJY1+JmZkZ9PX1cfv2bVy/fh2zZs3C2rVrERISgjVr1lTLN0x5qOc/SJH6dZ85cwZhYWHo168fHjx4ACMjI9y4cQNLly4VuEJhZGdnw8jICM7Ozjh+/Dg2bdqEdevW4c8//5TkL/+goCDNFR1Wr16Npk2bYtmyZcjPzxe4sqql/rwsXLgQ+vr6WLFiBVq3bo2QkBDMnj0bBQUFAldY9fLy8pCfn48pU6Zg0aJF6NGjB7Zt24Zp06Zh2rRpmpNSS42Ojk61/kFbfV9ZFbCyssLw4cPx1ltvoUWLFrhw4QLOnj2LgwcPwt/fH3379hW6xCqj/kJNSUlBrVq1oK+vj6KiIklNyC1O/dpPnz6NLl26wNTUFK6urgCe9LDevXtX4AqFUbNmTUyePBmFhYW4efOmZnl/cnKypJb3qz8X165dw6JFi3Dq1CncuXMHv/76K9atW4f79++jXr16whZZhYofJ4YMGQJra2s0b94c77//PgYNGoRr167B3d1dwAqrTvFz+T18+BDffPMNCgsL0aJFCxgZGSEuLg4ZGRnYvn275OYdSmEaSPV8VVVE/eHp0qULAKBz587o3LkzAOmtEFLbtWsXFAoFJkyYIMnXr6Z+7bm5uWjSpAn++OMPzcrTc+fOoX379kKWJxh1j4iurq6kl/cDT+Zc6unpYefOndi/fz9GjhyJNm3aYO7cuZpTyUiBOognJiZCpVIhMjISXbt2BfDkSzgrK0tSAVX9oyQnJwf6+vrIzMyEsbGxZrtMJoO/vz927NghZJlVTirTQKrXq6liOjo6mmCmHqpTqVSQy+WSCyTqA0abNm2wcOFCnD17Fp9//jmaN28ucGXCGjJkCKZMmYIrV67A29sbd+7cwY0bNzBu3DihSxMEl/c/UVRUBGtrawwbNgyHDh1Co0aN0KFDB5w4cQLm5uaSOn+d+t88Li4O4eHhOHbsGDw8PODu7o78/HwMGjRIUgse1O3h6+uLI0eOYP78+WjZsiXi4+Nx9uxZTJkyBeHh4ZoTp1Zn6sBefBrIhg0bYGRkhMuXLyM0NBTTp08XusxKxdWXlUgqQy8vo1AosGHDBqSmpqJv376a87RJpX3UQf3SpUtIT0+Hg4MDdu7ciTNnzsDU1BTTp0+XzLVQn6f48n4pTOwvTv261b/01QoKCrBt2zYYGRlJaupDcSqVCjExMThx4gSOHTuG27dvQ0dHBw0bNsRnn30muYVT6enp2Lp1q6YXsW3btujatSsmTpyIr776qtovDlGPOK1cuRK2trawtLTEgQMHMG/ePOzevRv//vsvVq9eLXSZlYqh7BXk5OTg4cOHkupaf5ns7GzUrFkTCQkJ+Pzzz3Hr1i189dVXkvqSUYeyRYsW4a233sKIESOQm5urtaRdKgEV4PL+5/H394ednR3atm2L9u3bw83NDYB0pz7k5OQgKSkJOjo6sLS0hJGREbKyshAVFYW9e/eiX79+mksuSUFCQgIeP36MRo0aaa6vrJaUlKS5GLkULFq0CD179tRMA2nZsiW+/vpruLi4YMiQIUKXV6k4fFkB6i/dffv2ITw8HEuWLEFMTAz+/vtvtGnTBp06dRK6xCqlbo/NmzcjNjYWJ0+ehLu7O5o1awZLS0v8/vvvuHPnDsaPHw8DAwOhy33t1D0/2dnZqF27NgBINpABXN5fmqKiIixevBiRkZH477//EBQUBABo2bIlFixYIHB1VUd97IiLi8O6deuwb98+zdnZe/fujW7dusHLy0tzebbqTh3If/zxR8TExCA7Oxv5+fmwt7eHvb09evfuDVtbW1hbW0vqOCKlaSAMZRWg7lw8duwY+vXrh0ePHmHz5s24fv06IiMjUadOHXh4eAhcZdVRh5C4uDg0b94cQ4cORXp6Otzd3WFgYIBHjx5h5MiRCAgIkMwlp7KysnDlyhUEBQVh79696NixI7y9veHk5CSZA2lxxZf3u7u7IyYmBmvXrsWePXvg7u6Or776CiYmJkKXWSXU8+g8PT3h4eGBwYMH48KFC9i4caPmihdSoT6W/vnnn9DR0cF///2HBw8eICQkBH/88Qfc3Nyq/RBdceofMHv27MHUqVNhZ2eHlJQU3Lx5E2fPnoWfnx9sbW0lcYLlZ6eBLFy4EDt37sTGjRthamqKuXPnVstrwzKUVYD6g5OQkID69esjKCgIdnZ2+PbbbzF+/Hjk5OQIXKEw/ve//6GgoKBEV7upqSlsbW1Rv359gSqrenp6eli6dCkSEhJw8eJFRERE4I8//kCTJk3w/fffC11eleHyfm3q3o3k5GRYWlpCJpNBLpejVatWOHjwoORCmTpYGBoawt/fHwBgY2ODjz/+GJGRkTh9+jTs7e0lEULU7434+Hg0a9ZMs5IfAFq0aIHu3btrhiyre1sUt3//frz11lvw9fXFxIkTq/2oA0NZBSkUCvj6+uLLL79Eeno6tmzZgpycHCQmJqJVq1ZCl1dl1N3tFy5cQEREBBITE2FsbAxnZ2fN/woLCzFr1izJrChTqVSoUaMG7O3tUb9+fXTs2BFKpRJxcXGSOwkml/drk8lkyM/Px6hRo2Bqago3Nze0bdsWbm5uOHLkCAYOHCh0iVVKJpMhNzcXR44c0VzZQC6XIycnB0qlUjLDlsDTgHHt2jWcO3cOv/zyCwICAlC3bl0YGRnByMhI6BKrlFSngXCifzmpJ7KrXbhwAbVq1YKTkxMOHjyI4ODgarca5EXUv2DVJ9F1dnZGZmYmUlJSkJiYiLFjx6Jt27ZCl1nlzp49izNnziA9PR12dnZo3bo16tWrJ8kL1ANAWloaZs+ejZo1a5a6vN/U1BTDhw8XuswqoVAoEBMTg9jYWFy8eBFRUVF48OAB3n//fXzxxRdCl1flkpOT8e233+Ly5ctITU2Fvb09DA0N0axZM8yaNUvo8qpcWFgY9u7di4cPH8LAwAAmJiawtLTEkCFDJDWUCzyZBjJixAhcu3YNHTp00JoGUl0xlJXT2LFjsWDBAvz999+wt7dHx44dNT1AqampyMrKgqOjo8BVVq3c3FyMHTsWmzZtAvDkCzgpKQlxcXHo2LEjjI2Nq+2vmuLUr/H69euYO3cuXFxccOLECXh6eiIyMhL16tXDmjVrqn07PI/Ul/cX9+znQaFQaHpYpSwmJgbHjx/HuXPncPPmTSQlJWHs2LGYOHGi0KVVGfXoQ1paGmJiYnDjxg1ER0dj4sSJ1XIO1Yvk5+cjMTFRMw3k+vXruHHjRrWeBsJQVk4ZGRmoXbs2vv76a5w+fRoZGRmws7ODr68vfH19JbVkW/3FEh0djeXLl2Pq1Klo1KiR0GUJRn0w/eWXX2BoaIiGDRtix44d+PLLLzFv3jx06tRJcsNTalze/9SxY8dw8OBBpKeno1mzZprhSymLioqCtbU1LCwsNNtycnJw8eJFGBsbw9PTU8Dqqk5RURFiY2MREhKCevXqoXPnzjAzM5PcZwQo/Xx+xaeBuLi4CFzh68FQ9gpUKhXu3r2LM2fO4PDhw4iLi8O+ffuELqvKqD80oaGh+Omnn2BkZIS2bduibt26cHR0hLu7u6TmQajbY+zYsZgwYQL27t2LJk2aoHfv3vj+++9hYWEhmSE6oOzL+4HqOz/kWffv38eAAQMwduxYFBYW4tKlS7h79y5yc3Oxa9euEudvq+4yMjKwbt06xMXF4ebNmzAzM0ObNm3g6ekJX19focurMurPysaNGxEeHo7atWvjxo0buHfvHqZNm4Y+ffpI5jNSnBSngXCi/yuQyWRwdHSEo6Mj+vfvj8LCQqFLqlLqA4SHhwe++uor3L59G3fu3MHFixdx4MABfP7555LqAVC3R+/evWFvbw9LS0scPnwYxsbGOHz4MBYtWiRwhVWLy/ufUn+hXrp0CV27dsXw4cORk5ODvn37Ii0tDampqZIKZOp/8wMHDiAlJQUtWrRAbm4u2rRpowlpvr6+kgki6vf/v//+iwkTJmjm4Z44cQLbtm1Dy5Yt4eDgIGSJVab4NJDvv/9eaxrIli1bqv00EIaySlTdLoxaVuqejzZt2qCwsBD379/HzZs3JTuUGRAQAAAYOHAgEhMTsWfPHnh5ecHV1VXgyqoOl/drU3+BODo64vbt2wCg6UU2NTWV1OligKfnJzt9+jRGjRqF48ePw8vLCx999BGSk5PRvn17zf2q65dvcTKZDNnZ2cjOztZaGNWhQwcsW7ZMUnMNi4qKIJfLERYWhm7duqFhw4ZIT0/H5MmTNdNAqvN7Qpopgl6Z+pfuvXv3cPDgQcTHx8Pc3BwNGzZEw4YN4eXlJZlTYDyPiYkJxo8fj4KCAs0wnVRweX/prly5gi1btmDLli3w9vZGx44d0bZtW625VFKg7kXNzMyEpaUlbt68if79+wMAbt68ibfffhuAdC5Qr1Qqoauri5YtW2LYsGEYN24cHB0dkZqaisLCQknNJ1P/SDt79qxmGkinTp3w1ltvoWHDhlAoFAJX+HpxTlk5HTlyBLdv38aHH34odCmCUs+BmDFjBmxsbLB3717Y2NggLS0Nurq6mDx5Mnx8fIQukwTG5f1PA+rVq1cxduxYLF++HFlZWTh79izOnz+PpKQkHDhwQBI9hs+6d+8ezMzMsH//fqxatQpubm64ffs2QkJCJHn9z9TUVKxbtw5paWmIjo6GiYkJRo8eDV9fX8ldEzU0NBTe3t4ICgpCZGQkevXqhRUrVmDRokXVetSBPWVldOfOHVhZWeHIkSOalUDqVSFnzpyBsbFxtX6jPEt9cLh27RoWLlyIyMhIzJkzBw8fPsTy5ctRt25dANIZfqDSdezYEb6+viWW90sxgCQnJ+PDDz9Eq1atUFBQgE6dOiE3NxdZWVmSbA8AmlM89OnTBwYGBrh37x4GDx4MuVwumWPHjBkz4OHhAW9vbzRo0ABffvklYmNjYWhoCCMjI82JU6UUyADpTgNhKCujCxcuYO7cucjLy8O9e/dgZGSE1q1bw9LSEitXrsTgwYOr/ZvlWTdv3oSpqSmysrKQmpoKBwcHODg4oEaNGqhXrx4A6Qw/AOxFfVZRURFu3bqltby/Xbt2klver54jc+DAATx+/BgPHjyAjY0NAKBmzZrVeiVZadRhKy4uDsHBwdDT08M777yD7t27l7hPdZeVlQVLS0tEREQgKCgI+vr6cHd3R/v27eHh4aEJZFImtWkgHL4sh0ePHmHKlCmoV68eDh8+jMTERDg6OsLW1harVq2S3ME1JSUFt2/fRv369bFw4UJ4enri0aNHmus8SmFVHfC0FzUwMBCenp54//33Jd2LyuX9pQsMDMThw4fx8OFDNGjQAB07dkSnTp0kdVk2tcLCQvTp0wdeXl64efMmLl68CLlcDgcHB6xbtw516tQRusQqo1AokJ2djeTkZNy8eRORkZG4ceMG4uPj0aZNG8ybN0/oEqkKMZSVkfpC2/n5+ZqVMPn5+Thz5gysrKzg7OwscIXCCgsLwy+//ILGjRujc+fOmus9SqHLPSQkRNOL2q5dO/Tr10/Tizps2DAMHjwYPXr0ELrMKqMOXMOGDSt1ef+XX34pmeX9pblz5w7OnDmDI0eOIC4uDqGhoUKXVOWioqKwePFibNiwQbPt6tWrOH78OMaNGydcYQJTqVRQKBR4/Pgx7t69C319fXh4eEjmWEoMZWWi/pLJz89HaGgocnJy4OXlhYYNGwpdmqAyMzNx4sQJ1KxZE61atYK+vr5kTwvCXlRt2dnZGDZsGIKDg7W29+3bFz/99JOkhi9fpPjZyqXg999/h5GREWrWrInk5GR88MEHQpckClIZVSgLqU8Dkc7R4BWoh1kmTpyIxo0bY9OmTTA0NISJiQns7e2xePFimJubC1xl1VAfPC5duoQFCxbAzs4OBQUF+P3332FhYQF3d3dJnbUeeNKLampqitWrV6NGjRqYNWuWVi+q1AIZl/eXnZQCGQDk5eVh586dSE9PR0ZGBs6ePYv3338fbdu2ldS5uJ7FQMbFdGrSOiJUgLqX7MqVKygqKsLAgQNx/vx5rFy5EjNmzECNGjUkE8iApyd9PHjwINq0aYORI0fi/v37ePDggdaqOqn88lOpVJph7Wd7UTt27Ch0eYKQy+WQy+UYP3481q1bh9DQUM3y/smTJwMAh2Mkaty4cRg3bhySk5Nx8eJFHD58GIGBgUhPT0ft2rURFBQEY2NjocusEupj5PXr11GzZk3Y2dmhqKgIgDRDGhfTPcHhy5dQf3DUF5l2dHTE7t27sXTpUuzbtw93796V5ByIFStWlLgAe2ZmJnR0dCTXMwQAY8eOlXwvKlByeT+AUpf3S4nUh2PKIi4uDleuXEHPnj2FLqXK/fLLL0hOTsbIkSM1pwiRKk4DYU/ZS6l/sRgYGMDd3R2JiYkwMTEB8OQSIZaWlkKWJwiVSgUfHx/MmjULvXr1Qrt27eDm5qZpF6lgL6o2Lu/XxuGYsqtXr57mNDpS061bNwQFBWHo0KFo2bIlPvjgA7i7u0uut4zTQJ5gT1k5KBQKpKen4/PPP8fNmzfh6uqK+fPnS+bM5OpeQ/XZt93d3SGTyfDgwQPk5eWhZcuWmDp1qtBlVhn2opbE5f1PcVUuPU/xxWPZ2dmoVasWMjMzsWLFCly/fh2DBg1Cr169hC6zynAx3VPsKSsHfX19WFtbY+vWrbhx4wZq164tyZ6yS5cuYfDgwRg6dCgSExORnp6Omzdvan7JSGW+EHtRS9LX14e+vj7q1KmDRo0aoXPnzlrL+wHpvD969+6Nt99+WzMcs2TJEq3hGF9fX6FLrDLqL91Hjx7h/v37cHFxEbokQalPKPzTTz/hn3/+ga6uLkxMTODg4ICEhASEhYWhV69ekvmscDHdUwxlL6D+QERERGDdunWIjY1F+/bt4evrC09PT5ibm0vqJJjqEKJQKDTnZbO1tYWtrS1cXFw0iwCkcBApbsSIEVAoFLC1tcWmTZvQqlUruLq64qOPPhK6NEGoexBlMhlq1KgBS0tLrYAqlfcHh2OeKn5Vg3PnziEwMBAKhQL6+vrIz8+Hrq6uZN4XwNPPgJ2dHWxsbBAQEIA2bdrAwMAA33zzjeY7RQpDmJwGoo3Dly+gnv/RvXt3jBs3DoaGhjh27BguXryI2NhY/Pzzz5K76HZGRgYGDRqEpKQkdOvWDR06dECbNm0k2Sv0PFLuRaUnOByjbf/+/di3bx/i4uLQv39/DBs2THPbypUrUbt2bYwYMULACoWTmJiIdevW4eHDh+jatSvatWsnqRDCaSDa2FP2AupzCDk6OqJ3794AAH9/fwBAUlKS5CYuA0+G6pYtW4bbt2/jypUrCA4OxooVKyQ3X4i9qNq4vF8bh2O0eXl54e7duzh16hQCAwPxww8/oFWrVujevTtOnDiBSZMmAZDGqXTUr/HKlSv4559/0Lx5c3h5eSEoKAgzZszA4MGDMWvWLKHLrDKcBqKNoew5Nm/ejJ9//hndunWDhYUFIiIi4OXlpbldqifANDAwgKurK1xdXdGlSxfJzhdSdzDPmTNHqxd1yZIlkuxFVR9Yw8LCJL+8n8MxJdWpUwdjx46FnZ0devTogfDwcOzfvx+//fYb3nnnHXh7ewOQRoBXv8aUlBTcu3cPUVFRsLOzQ5s2bWBqagorKysA0rvaA6eBPMHhy+fIyspCcHAwwsLCcOnSJeTm5qJt27aa6zpKZcXls6TwS7Y8Pv74Y6xZs0Zrm7oXVYpnKL979y6CgoKwe/duyS7v53BMSeofa9nZ2SXm0kmpR/lF1NdXpiekOg2Eoew5duzYgf79+2v+vnr1Kvbv349jx44hJiYG69ev1/y6I2kp3ouak5ODd999V6sXVWq4vL90GzduhIeHBxITE/Hff/9hzpw5mD17NiwtLTFhwgShy6sy6pC6d+9enD59GqdPn4aNjQ18fX3RoUMHODs7S+7HnkKhwLFjx3DgwAGYmJjA0dERffv2ldTiD4DTQErDUFaKpKQkTJs2DQYGBujduze6du1aohtZSm8UzhfSxl5UbeoD6/Lly0ss7z916hRatWqFZcuWSWZouzipn9tQTaFQoE+fPpg9eza+/vprdOvWDVu3bkVGRgb++ecfODo6Cl1ilVB/BrZv344jR46gXr16sLW1xfHjx2FqaopvvvkGhoaGQpdZZbiYriSGsue4ffs2duzYgSNHjsDd3R3vvvsuOnToIMkQosbLgTzBXtTS7dixA6GhoVrL++vUqQOZTAY9PT1J/ZApjRSHY9Q/6E6cOIE9e/bgyy+/xIQJE7B161Zs3boVBQUFGD58uNBlVhl1e4wfPx6jRo1CmzZtADxZ1f7VV19hyJAhkjx2cBrIU9JNGC9Rv359TJs2DZs2bYKtrS02b96MNWvW4N69e0KXJphu3brByMgIQ4cOxZQpUxAZGanpMZOKpKQk/P333xg3bhz27duHwsJCuLq6YvLkyQgJCUFMTAzatm0rdJmC6N+/P+bPn49r165h6dKlOHPmDDIzMzXzZKQQyJRKJQAgIiIC48aNQ5cuXTB79mwcOXIEZmZmsLS0hJR+B6v/za9duwYvLy+cP38ejRs3BvBk8n9kZCQASOY4ov5RX1hYiOTkZABPRl1q166N9PR0zYIpKbxHNm/ejE6dOmHBggWaxXTFWVtbSy6QAewpK0GpVOLixYtISkpCbm4u3N3dkZeXh6CgIGzfvh0mJiYICQmRTE8R5wuVxF7Up0pb3q9UKhEUFITjx49Lbnk/h2NKd/36dRgYGCAvLw8zZsyAs7MzEhIS0Lt3b/Tr109yQ9vHjh3D2rVr8d5778HNzQ2nT59GaGgotm/fLnRpVYbTQErHUPaMkydPYvTo0ZpTYURHR6NBgwZo1KgRYmNjERUVhZ07dwpdZpXhfKHnS0lJwaZNmxATE4OmTZvivffek0xYf1ZYWBj++usvZGRkwM7ODvXr18e1a9fg5OSEsWPHSm55P4djnu/o0aPYt28fXFxcMHDgQMnMoXr8+DF0dXVhZGSEgoIC7Ny5E9u3b0dmZiZ8fX3RtWtXeHl5SWbRA6eBlI6h7BlZWVnYuXMnoqOj4ePjA39/fyiVSuTn52tOaCdFnC/EXtTykOLyfq7K1aY+JuTk5CA8PBwFBQVwc3OTbA/IggULYGJiAldXVzRo0AD169cH8KS3WalUSuY4CnAx3YswlJUiLy8PERERCAoKQmZmJrp06YLu3bvDzMxM6NIEJfXLgbAXtXRc3v8Eh2O0qYP5okWLEBMTg7t37yI3Nxc2NjZwcnLC6NGjJXNhcqVSqelVT0hIwKNHj+Du7o5GjRqhSZMmsLKyQv369SXRQ6bGaSClYygrRqlUQqFQaHWnX7hwAZcuXcKjR4/g7e2tWS0jBZwvpI29qNq4vF8bh2NK5+/vj82bN8PCwgL5+fk4efIkgoODMWLECLRu3Voyw3Vqx44dw7Jly9C3b1/8999/SEpKgoODA5YsWSJ0aYLgNBBtDGXFnDt3DsuWLcOjR49gZ2cHe3t7PHz4EBcuXEBycjKsrKxw7Ngxocuscpwv9BR7UZ/i8v6nOByjLSEhAbdu3YK7uzv+/PNPDBs2TJLXCi5OoVBAX18f33//PerUqYORI0cCeBJKHjx4AHd3d0kEVE4DeTGGsmJiYmIQExMDY2NjxMbGAgBsbGxQo0YNxMfHo27duujevbvAVQpPivOF2Iv6fGPGjEHv3r3Ro0cPTfAYNmwYJk2ahJYtW0omjHA45qmDBw9iwoQJ0NfXh1wuh5eXFyZPnqw5HYbUKBQK6OnpQSaTYcSIEfjyyy/h4eEhyQVSnAbyYgxl9EKcL/QEe1Gfj8v7tXE4BpprXCYlJeHIkSPYt28fLly4gBo1aqBu3bqYO3cuPD09hS6zSsTHx2Pq1Knw8/NDw4YNsWLFCvz1119ClyUYTgN5MYayMpJCt3JxnC+kjb2o2ri8/ykOx2grKCjA1KlT4e/vj4CAAK3bYmJiEBQUhP79+6Nx48aSeH9kZmZi27ZtuHTpEq5fv46MjAz4+vrCzc0NzZs3R/369WFsbCx0mVWK00Cej6GMSsX5QvQiXN7/FIdjtGVlZWH9+vUIDg6Gjo4ORowYgS5dusDW1lbo0kQhOjoaBw8exPnz53Hx4kVMmDABo0ePFrqsKsFpIC8njdnZVG7luRyIFL54X0QKv/aLUyqVsLW1RUxMDCIiIiS/vL9p06aYMWMGoqOj0aJFC8ycOVPSwzHGxsaYOHEiJk6ciK1bt+K///5DWloa+vXrJ7nTgpSmSZMmaNKkCYAn00NycnIASOM4cvHixRdOAwkKCpLsNBA19pTRC3G+EL0Il/c/weGYpxQKBTIzMzXnMDx58iR+++03REVFwc/PD1988QWsra0FrpKEwGkgL8dQRiVwvhC9DJf3P8HhGG2pqamYPHky6tati7t370JXV1ezAnXbtm2Ii4vDwYMHUbduXaFLJRIlhjIqgfOF6EW4vP8prsrVduPGDUyaNAkeHh4YOXIkVCoV7ty5AyMjI1hbWyM5ORnt27cXukwSISn8iCsLhjLSwsuB0Itweb82DseUdPPmTc1cMl9fX/Tq1QtOTk6a2/mDjuj5GMrouThfiJ7F5f1UVklJSTh06BAyMzPh6enJ1dpEZcBQRiVwvhCVlZSX97+M1D4j6enp+Ouvv2BhYQF7e3vk5OQgNjYWFy9exPnz52Fubo6NGzfCwMBA6FKJRIunxCAt6vlCwJMJy19++SWAJ8OaFhYWsLCwAABJfdnQ80l5ef/LSO31R0dHIzw8HCkpKUhOToabmxtq166NxMREZGVloX79+jAwMODwJdELMJSRxrPzhTIyMuDh4QEAkpvATeWnr6+vOX+d1AIJAV5eXmjatKnmEmz379+HSqXSnDQ2PT0dAOeUEb0Ihy9Jg/OFiKiyMYQRlR1DGT0X5wsRERFVHYYyKhP1fKHatWtzvhAREdFrwFBGREREJALs7iAiIiISAYYyIiIiIhFgKCMiIiISAYYyIiIiIhFgKCMiIiISAYYyIhK94cOHY/78+SW279q1C61atRJNPa/bDz/8gPfee6/Kn5eIqgZDGRHRcxQUFAhdAhFJCEMZEVUbEREReP/999GsWTO0atUKgwYNwr179zS3Hzx4EH369IGHhwc6d+6MVatWobCwUHO7s7Mz/vzzT3z88cdo1qwZ1qxZU6bn9fPzw5o1azBz5kw0b94cvr6+2LZtm+b2hIQEODs7Y+/evRg0aBA8PDzQs2dPnDlzRnOf0nr9Dh48CGdnZ83tq1atQkxMDJydneHs7Ixdu3ZVqJ2ISJwYyoioWigsLMSnn36K1q1b46+//sK2bdswcOBAzXUXz549i+nTp2PEiBEIDQ3FN998g127dpUIXqtWrULXrl2xZ88e9OvXr8zPv379eri7uyMkJARDhgzBnDlzcOvWLa37fPfddxg1ahRCQkLQrFkzfPzxx5oLdb9MQEAAPvzwQzRq1AgnTpzAiRMnEBAQUOb6iEj8GMqIqFrIyspCZmYm3n77bTg4OMDJyQl9+vSBra0tgCdha+zYsejTpw/s7e3Rvn17fP7559i6davWfnr27Il+/frB3t5e89iy6NSpE4YOHQpHR0eMGTMGderUQUREhNZ9hg4dCn9/fzg5OWHOnDkwMTHBzp07y7R/AwMDGBkZQS6Xw9LSEpaWljAwMChzfUQkfrpCF0BEVBlq166Nvn37YvTo0Wjfvj28vb3RvXt3WFlZAQBiYmJw/vx5rZ4xpVKJ/Px85ObmwtDQEADg7u5eoedXDzMCgEwmg4WFBVJTU7Xu07x5c81/6+rqwt3dvURvGhFJF0MZEYlezZo1kZWVVWL748ePYWJiovk7MDAQw4cPx/Hjx7Fv3z4sX74c69evR7NmzZCTk4OJEyfinXfeKbGfGjVqaP7byMioQjXq6mofTmUyGcpzaWEdHZ0S9+dCAyJp4fAlEYle/fr1ERUVVWL71atXUa9ePa1trq6uGDduHLZu3YrGjRvj77//1my/ffs2HB0dS/xPR6dqDoUXL17U/HdhYSGioqLQoEEDAECdOnWQnZ2NnJwczX1iYmK0Hq+np4eioqIqqZWIqh57yohI9IYMGYLNmzdj3rx5eP/996Gvr4+jR49i7969+OmnnwAA8fHx2L59O/z8/GBlZYXbt28jLi5Oc16vTz/9FB9//DFsbW3h7+8PHR0dxMTE4Pr165g0aVKVvI4///wT9erVQ4MGDfD777/j0aNHmsUETZs2haGhIZYtW4YRI0bg0qVLJVZX2tnZISEhAdHR0bC2toaxsTH09fWrpHYiev0YyohI9Ozt7fHHH39g+fLlGDVqFAoKCtCgQQOsWLECnTp1AgAYGhri1q1bCA4ORkZGBqysrDB06FAMGjQIANCxY0esWbMGq1evxq+//gpdXV00aNAA/fv3r7LXMWXKFPzyyy+Ijo6Go6MjfvrpJ5iZmQF4Midu8eLF+O6777Bjxw54e3tj4sSJ+OqrrzSP9/f3x4EDBzBixAg8fvwYgYGB6Nu3b5XVT0Svl0xVnkkPRERUbgkJCejcuTNCQkLQpEkTocshIpHinDIiIiIiEWAoIyIiIhIBDl8SERERiQB7yoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhEgKGMiIiISAQYyoiIiIhE4P8BhMEseqFs4foAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_metric('faithfulness')" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "cd595fed-e1fe-4b4f-9ef8-b2cb9a625d2b", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAHMCAYAAACUdN+cAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABruklEQVR4nO3dd1RTd/8H8DcEkCGKTAUBJxvEBeJE3LPuidbxVO1Pbau2anetVrSPWrX6tLVu6xZBLeDAPXBWRVFQUQSUIlPZI8nvD0+iEVSCwI3c9+scz5Gbm+STL5fkne+4V0sul8tBRERERILSFroAIiIiImIoIyIiItIIDGVEREREGoChjIiIiEgDMJQRERERaQCGMiIiIiINwFBGREREpAEYyoiIiIg0AEMZERERkQZgKCOiUs2dOxd+fn5Cl6FxcnJy8PXXX6Ndu3ZwdHTETz/9JHRJFW7v3r1wdHTEjRs3hC6FSFQYyqha2Lp1KxwdHTF06FChSxHEmDFj4OjoqPzn4eGBfv36YePGjZDJZEKXV6388ccfCAoKwsiRI/Hzzz/jgw8+qNTn8/PzU/ndvvxv4sSJlfrc5TV37lw0b978tbc3b94cc+fOrcKKXi8xMRGOjo5Yt26d0KUobd26FXv37hW6DBKAjtAFEFWEAwcOwMbGBpGRkXj48CHs7e2FLqnK1a1bFzNnzgQAZGRk4O+//0ZAQAAyMjIwY8YMgaurPs6fP49mzZph2rRpVfaczs7OGD9+fIntlpaWVVYDVZ3t27ejTp06GDRokNClUBVjKKP3XkJCAq5evYpVq1bhu+++w4EDB6r0A7MqyGQyFBUVoUaNGq/dx9jYWKXXZuTIkejVqxe2bNmCTz75BBKJpCpKrfbS0tLQpEmTCnu84uJiyGQy6OnpvXYfKyurSu+Rq85yc3NhaGgodBlEb8XhS3rvHThwALVr10anTp3Qo0cPHDhwoMQ+Lw9R7Ny5E127doWbmxsGDx6MyMhIlX1TUlLw5ZdfomPHjnBzc0P79u3x8ccfIzExEQAQEBAAb29vyOVy5X3mz58PR0dHbN68WbktNTUVjo6O2LZtm3JbYWEhVq5ciW7dusHNzQ2dOnXCzz//jMLCQpUaHB0d8eOPP2L//v3o06cP3N3dcfr0abXapUaNGnBzc0NOTg7S0tJUbtu3bx8GDRoEDw8PeHl5YcaMGUhKSnrrY8pkMmzcuFFZU9u2bfHdd9/h6dOnyn0mT56MLl26lHr/4cOHq3z7DwwMxNixY+Hj4wM3Nzf07t1bpb0U/Pz8MHnyZFy+fBlDhgyBu7s7unTpguDg4BL7Pnv2DAsXLoSfnx/c3NzQsWNHzJ49G+np6cjJyYGnpycWLFhQ4n7//vsvnJ2d8ccff5Ra+4ULF+Do6IjExEScOHFCOYSoOC7S0tLw1VdfoW3btnB3d0f//v0RFBSk8hgvH4cbN25E165d4e7ujtjY2FKfUx3R0dGYO3cuunTpAnd3d7Rr1w5ffvklMjIySuybnJyMr776Cu3bt4ebmxv8/Pzw/ffflzgOCwsLERAQgDZt2sDT0xNTp05Fenr6O9f6qqKiIqxatQrdu3eHu7s7vL29MXLkSJw9e1Zlv9jYWHzyySfw8vKCu7s7Bg0ahKNHj6rso5gPd/HiRfzwww/w8fFBp06d1KpH8RhXrlx56+tXHJtnzpzBBx98AHd3d/Tu3RuHDx9W2e/XX3+Fo6Pja59LcRz5+fnh7t27uHjxovIYGzNmjFr10/uLPWX03jtw4AC6desGPT099O3bF9u3b0dkZCQ8PDxK7Pv3338jJycHw4cPh5aWFtauXYvp06cjPDwcurq6AIDp06fj3r178Pf3h42NDdLT03H27FkkJSWhfv36aNWqFTZu3Ii7d+/CwcEBAHD58mVoa2vj8uXLGDt2rHIbALRu3RrA80Dz8ccf48qVKxg2bBgaN26MO3fuYNOmTYiLi8P//vc/lVrPnz+PsLAwjB49GnXq1IGNjY3abfPo0SNoaWmhVq1aym2//fYbVqxYgV69emHIkCFIT0/HX3/9hdGjRyM4OFhl31d99913CAoKwqBBgzBmzBgkJiZi69atuHXrFrZv3w5dXV306tULc+bMKfE7ePToEa5du4bZs2crt23fvh1NmzaFn58fdHR0cPz4ccybNw9yuRyjR49Wee6HDx/i008/xZAhQzBw4EAEBgZi7ty5cHV1RdOmTQE8n4Q/evRoxMbGYvDgwXBxcUFGRgaOHTuG5ORkODs7o2vXrggLC8OXX36p0nv4999/Qy6Xo1+/fqW+9saNG+Pnn39GQEAA6tatqxxONDU1RX5+PsaMGYP4+HiMHj0a9evXx8GDBzF37lw8e/YMH374ocpj7d27FwUFBRg2bBj09PRQu3btN/4ei4uLSw1DhoaG0NfXBwCcO3cOCQkJGDRoECwsLHD37l3s2rUL9+7dw65du6ClpQXgeSAbMmQIsrKyMGzYMDRq1AjJyck4dOgQ8vPzVXrsFixYgFq1amHatGl49OgRNm3ahB9//BHLly9/Y73qWrVqFf744w8MHToUHh4eyM7Oxs2bNxEVFYV27doBAO7evYuRI0fCysoKH330EQwNDREWFoapU6fi119/Rbdu3VQec968eTA1NcXUqVORm5tbrrrK+vrj4uIwY8YMjBgxQnlsfvrpp1i7dq2y/rL66quvMH/+fBgaGmLKlCkAAHNz83LVT+8hOdF77MaNG3IHBwf52bNn5XK5XC6TyeQdO3aUL1iwQGW/hIQEuYODg9zLy0uemZmp3B4eHi53cHCQHzt2TC6Xy+VPnz6VOzg4yNeuXfva50xLS5M7ODjIt27dKpfL5fJnz57JnZyc5J988om8bdu2yv3mz58v9/LykstkMrlcLpcHBwfLnZyc5JcuXVJ5vO3bt8sdHBzkV65cUW5zcHCQOzk5ye/evVumdvD395f37NlTnpaWJk9LS5PHxsbKFy9eLHdwcJBPmjRJuV9iYqLc2dlZ/ttvv6ncPyYmRu7i4qKyfc6cOfLOnTsrf7506ZLcwcFBvn//fpX7njp1SmV7VlaW3M3NTb5o0SKV/f7880+5o6Oj/NGjR8pteXl5JV7LhAkT5F26dFHZ1rlzZ7mDg4NK26WlpZV4nhUrVsgdHBzkhw8fLvG4it/D6dOn5Q4ODvKTJ0+q3N6vXz+5v79/ifu9qnPnziptKpfL5Rs3bpQ7ODjI9+3bp9xWWFgoHz58uNzT01OelZUll8tfHIctWrSQp6WlvfW5FM/n4OBQ6r8//vhDuV9pbfn333+XaLfZs2fLnZyc5JGRkSX2V7RRYGCg3MHBQT5u3DjlNrlcLl+4cKHc2dlZ/uzZszfWPGfOHLmnp+drb/f09JTPmTNH+XP//v1LtOmrPvzwQ3nfvn3lBQUFKvUOHz5c3r17d+U2Re0jR46UFxcXv/Ex5fIXv5OX/+bVef2K38+hQ4eU27KysuTt2rWTDxgwQLlt5cqVcgcHhxLPr3iuhIQE5bY+ffqU6Vik6ofDl/ReO3DgAMzNzeHt7Q0A0NLSQu/evREaGgqpVFpi/969e6v0SrRq1QrA83lpAKCvrw9dXV1cvHhRZUjuZaampmjUqJGyJ+yff/6BRCLBxIkTkZqairi4OADAlStX0KJFC2UPxcGDB9G4cWM0atQI6enpyn9t2rQB8Hx47GWtW7dWa+7S/fv34ePjAx8fH/Tq1Qvr1q2Dn58fAgIClPscOXIEMpkMvXr1UqnB3Nwc9vb2JWp42cGDB2FsbIx27dqp3NfV1RWGhobK+9asWRMdO3ZEWFiYyhBvaGgoPD09YW1trdym6OUBgKysLKSnp8PLywsJCQnIyspSef4mTZoof1/A899Dw4YNlb87ADh8+DCcnJxK9JoAUP4e2rZtC0tLS5Vh7jt37iAmJgb9+/d/fQO/walTp2BhYYG+ffsqt+nq6mLMmDHIzc3FpUuXVPbv3r07TE1Ny/z4zZo1w4YNG0r869Onj3Kfl9uyoKAA6enpaNasGQAgKioKwPPe2vDwcHTu3Bnu7u4lnkfRRgrDhg1T2daqVStIpVI8evSozLWXRa1atXD37l3l386rMjMzcf78efTq1QvZ2dnKYy8jIwPt27dHXFwckpOTS9T+rvMoy/r6LS0tVY65mjVrYsCAAbh16xZSUlLeqQYSFw5f0ntLKpUiJCQE3t7eyvkYAODh4YH169cjIiIC7du3V7lPvXr1VH5WBLRnz54BAPT09PD5559j8eLFaNeuHZo1awZfX18MGDAAFhYWyvu1atUKJ0+eBPB8mNLNzQ3u7u4wMTHB5cuXYW5ujujoaJUP6YcPHyI2NhY+Pj6lvp5X533Vr19frfawsbHBggULIJPJEB8fj99//x0ZGRkqiwPi4uIgl8vRvXv3Uh9DR+f1bwkPHz5EVlZWmerv3bs3wsPDcfXqVbRo0QLx8fGIiorCV199pXKfK1eu4Ndff8W1a9eQl5encltWVhaMjY2VP7/6uwOe//5eDs/x8fGvfW0K2tra6NevH7Zv3468vDwYGBjgwIEDqFGjBnr27PnG+77Oo0ePYG9vD21t1e+5jRs3BgA8fvxYZbu6v9s6deqgbdu2b9wnMzMTq1atQmhoaIljSRFw09PTkZ2drRzufZuXAzQA5dC24u/lXbwcdj755BP83//9H3r06AEHBwe0b98eH3zwAZycnAA8/73K5XKsWLECK1asKPXx0tLSYGVlpfxZ3TYuTVlfv729fYlA26BBAwDPj42X3zuI3oShjN5b58+fR0pKCkJCQhASElLi9gMHDpQIZa/75vxyj864cePg5+eH8PBwnDlzBitWrMCaNWuwadMmuLi4AABatmyJXbt2ISEhAZcvX0bLli2hpaWFFi1a4MqVK7C0tIRMJlPp2ZHJZHBwcMCXX35Zag1169ZV+fnlno+yMDQ0VPngbtGiBQYNGoRffvkF33zzjbIGLS0t/Pnnn6W2xZtWqMlkMpiZmWHJkiWl3v5yz0/nzp1hYGCAsLAwtGjRAmFhYdDW1lYJPfHx8Rg3bhwaNWqEuXPnol69etDV1cXJkydLPb9aRa4eHTBgANatW4fw8HD07dsXf//9N3x9fVVCYGVS93dbFp999hmuXr2KiRMnwtnZGYaGhpDJZPjPf/6jcnyr49WQqfC2x9PT00NhYSHkcnmJsCKXy1FQUKAyd61169Y4cuQIjh49irNnz2LPnj3YtGkT5s2bh6FDhyqPhQkTJqBDhw6lPqednZ3Kz29aqVxW5X39pXm1HRRK69En8WIoo/fWgQMHYGZmhu+++67EbUeOHMGRI0cwb968cn0A2tnZYcKECZgwYQLi4uIwYMAArF+/XhlIWrZsCQA4e/Ysbty4gUmTJgF4/uGyfft2WFpawtDQEK6uriqPGR0dDR8fn9e+QVckJycn9O/fHzt27MCECRNgbW0NOzs7yOVy1K9fHw0bNlTr8ezs7BAREYEWLVq8tU0NDQ3h6+uLgwcP4ssvv0RoaChatWql0pNx7NgxFBYW4rffflPpkXjTEGpZarx79+5b93NwcICLiwsOHDiAunXr4vHjx8rgWh42NjaIiYmBTCZT+SC/f/8+gJI9LhXt6dOniIiIwPTp01VOB/PqcKCpqSlq1qxZpjZ6FzY2NiguLkZ8fHyJcwY+fPgQUqm0xMIVExMTDB48GIMHD0ZOTg78/f3x66+/YujQobC1tQXwfEj4bT2GQnj48GGJAKpoe8XrfLmX7eXFNK/2ogKvD3BU/XFOGb2X8vPzcfjwYfj6+qJnz54l/o0ePRo5OTk4duyYWo+bl5eHgoIClW12dnYwMjJSOV2Ara0trKyssHHjRhQXF6NFixYAng9rxsfH4+DBg2jWrJnKcGCvXr2QnJyMXbt2lfp6yrtC7E3+85//oLi4GBs2bADwfC6TRCLBqlWrSnzbl8vlpZ4+QaFXr16QSqUlVokCz1cHvjqk07t3bzx58gS7d+9GdHQ0evXqpXK7oufr5TqysrIQGBio3ot8Sffu3REdHY0jR46UuO3V1/vBBx/g7Nmz2LRpE0xMTNCxY8dyP2/Hjh2RkpKC0NBQ5bbi4mJs2bIFhoaGyhW4leV1vYibNm1S+VlbWxtdu3bF8ePHS72EUnl71F6laMu//vqrxG1bt25V2QdAiePOyMgIdnZ2yr85MzMzeHl5YefOnXjy5EmJx6yM03So48mTJyrHXHZ2NoKDg+Hs7KwculT05L08vzA3N7fU07oYGBhUyBAxvX/YU0bvpWPHjiEnJ+e112b09PSEqakp9u/fj969e5f5cePi4jBu3Dj07NkTTZo0gUQiQXh4OFJTU1UmVQPPA1hISAgcHByUc9NcXFxgaGiIuLi4EqdW+OCDDxAWFobvv/8eFy5cQIsWLSCVSnH//n0cPHgQa9euLXXy9bto0qQJOnXqhD179uD//u//YGdnh88++wxLly7Fo0eP0LVrVxgZGSExMRHh4eEYNmzYay/d4+XlheHDh+OPP/7A7du30a5dO+jq6iIuLg4HDx7E119/rTI82alTJxgZGWHx4sWQSCTo0aOHyuMp7j9lyhSMGDECOTk52L17N8zMzMo9OXrixIk4dOgQPv30UwwePBiurq54+vQpjh07hnnz5innKAFA37598d///hdHjhzByJEjladEKY/hw4dj586dmDt3LqKiomBjY4NDhw7hn3/+wVdffYWaNWuW+7GB56ex2LdvX4ntRkZG6Nq1K2rWrInWrVtj7dq1KCoqgpWVFc6ePasy11Jh5syZOHv2LMaMGaM8NUtKSgoOHjyIbdu2vfGUKGXl7OyMoUOHYvPmzXj48KGyd+vcuXM4efIkhg4dqvK76NOnD7y8vODq6goTExPcuHEDhw4dgr+/v3Kf77//HqNGjUK/fv0wbNgw2NraIjU1FdeuXcO///6L/fv3v3Pd5dWgQQN8/fXXuHHjBszMzBAYGIi0tDSVRTbt2rWDtbU1vv76a9y/fx8SiQSBgYGoU6dOid4yV1dXbN++Hf/73/9gb28PU1PT187lpOqFoYzeS/v370eNGjVeew4gbW1t+Pr64sCBA2/s/XlV3bp10adPH0RERGD//v2QSCRo1KgRli9fXiJUtGzZEiEhIcqhTOD5RHlPT0+cO3dOZbuiptWrV2Pjxo3Yt28fjhw5AgMDA9SvXx9jxoxRezixrCZOnIgTJ07gr7/+wvTp0zFp0iQ0aNAAGzduxOrVq5Wvu127dm+9APmPP/4INzc37NixA7/88gskEglsbGzQv39/ZW+hQo0aNeDn54cDBw6gbdu2MDMzU7m9UaNGWLlyJZYvX47FixfD3NwcI0eOhKmpaYkFAWVlZGSErVu34tdff8WRI0cQFBQEMzMz+Pj4qAydAs/P/dSuXTucPHnync+Wr6+vjy1btmDJkiUICgpCdnY2GjZsiICAgAq5VM7t27dVzu+mYGNjg65duwIAli5divnz52Pbtm2Qy+Vo164d/vzzzxJzsKysrLBr1y6sWLECBw4cQHZ2NqysrNCxY8cKnev2448/wsHBAYGBgVi2bBkAoGHDhvjmm29KnINuzJgxOHbsGM6ePYvCwkJYW1vjs88+U/mC0KRJEwQGBmLVqlUICgpCZmYmTE1N4eLigqlTp1ZY3eXRoEEDfPvtt/j555/x4MED1K9fH7/88otK2+vq6mLVqlWYN28eVqxYAQsLC3z44YeoVatWiXmmU6dOxePHj7F27Vrk5OTAy8uLoUwktOQV1V9NRPSemTp1Ku7cuVPqcCdRWfj5+aFp06avvRIEkTo4p4yIROnJkycV0ktGRFRROHxJRKKSkJCAf/75B3v27IGOjg6GDx8udElERADYU0ZEInPp0iXMnj0biYmJWLRoEU/sSUQag3PKiIiIiDQAe8qIiIiINABDGREREZEGEN1Ef5lMhuLiYmhra/NSFkRERFTp5HI5ZDIZdHR0XntNVUCEoay4uLjUy4sQERERVSZ3d3fo6em99nbRhTJFQnV3d3/t9eKIiIiIKopUKsWNGzfe2EsGiDCUKYYsJRIJQxkRERFVmbdNm+JEfyIiIiINwFBGREREpAEYyoiIiIg0AEMZERERkQZgKCMiIiLSAAxlRERERBqAoYyIiIhIAzCUEREREWkAhjIiIiIiDcBQRkRERKQBGMqIiIiINABDGREREZEGYCgjIiIi0gAMZUREREQagKGMiIiISAMwlBERERFpAIYyIiIiIg3AUEZERESkARjKiIiIiDQAQxkRERGRBmAoIyIiItIAgoayS5cuYcqUKWjfvj0cHR0RHh7+1vtcuHABAwcOhJubG7p164a9e/dWQaVERERElUvQUJabmwtHR0d8//33Zdo/ISEBkydPhre3N/bt24cPP/wQ33zzDU6fPl3JlRIRERFVLh0hn7xTp07o1KlTmfffsWMH6tevj7lz5wIAGjdujCtXrmDjxo3o0KFDZZVJREREVOkEDWXqunbtGnx8fFS2tW/fHgsXLlT7saRSaUWVRURERPRaZc0c71UoS01Nhbm5uco2c3NzZGdnIz8/H/r6+mV+rBs3blR0eUREBEBXVxcuri7QkbxXHzFlUiwtxq2oWygqKhK6FKqGqt9fTBm5u7tDIpEIXQYRUbUkkUiwKGIn4p89EbqUCmNXyxJzfYbD1dVV6FLoPSOVSsvUGfRehTJzc3OkpqaqbEtNTUXNmjXV6iUDnr9hMJQREVWe+GdPcC/jsdBlVDh+dlBlea/OU+bp6Ynz58+rbDt37hw8PT2FKYiIiIiogggaynJycnD79m3cvn0bAJCYmIjbt2/j8ePn36yWLl2K2bNnK/cfMWIEEhIS8PPPPyM2NhZbt25FWFgYxo0bJ0T5oiCTyYQuoVJU19dFRPQ+kMur52K7d31dgg5f3rx5E2PHjlX+HBAQAAAYOHAgFi1ahJSUFCQlJSlvt7W1xR9//IGAgABs3rwZdevWxYIFC3g6jEqkra2Nn3/Zi/jEFKFLqTB29S0we8agct1XKpNBov1edTCXSXV9XUSkmbS0JDh/9yc8y4sXupQKU8vADm2afv1OjyFoKPP29kZMTMxrb1+0aFGp9wkODq7EquhV8YkpiL3/r9BlaASJtjZ+XBeEh0mpb9/5PWFfzxzfTRyo9v2qc5Crzq+NSFM8y4tHZu5docvQKO/VRH8iTfAwKRV3EhhSJdra+GbnXjx4Un0CKgA0tDTHguHl60klInoXDGUCWb16NYKDgzFgwABMnTpV6HKIyuXBk1TEPGZAJSKqCOyfF0B+fj6Cg4Mhk8kQHByM/Px8oUsiIiIigTGUCaC4uFi5+k8mk6G4uFjgiojoXUmr8Yre6vzaiDQJhy9fIZPKoC2pflm1ur4uIk0h0dbGl2F7cD+9es2xa2RqjoBeQ4Qug0gUGMpeoS3Rxn+/2YyEB8mV9hwylfOYaGHupFXQ1qq8M0TbNrTCFwvGvn1HInon99NTEf0k6e07EhGVgqGsFAkPkhEbnVipzyExMINULw2SQlM8iOGbOBERkdgxlAlEN68edPPqCV0GERERaQhOMiIiIiLSAAxlRERERBqAoYyIiIhIAzCUEREREWkAhjIiIiIiDcBQRkRERKQBGMqIiIiINABDGREREZEGYCgjIiIi0gAMZUREREQagKGMiIiISAMwlBERERFpAIYyIiIiIg3AUEZERESkARjKiIiIiDQAQxkRERGRBmAoIyIiItIADGVEREREGoChjIiIiEgDMJQRERERaQCGMiIiIiINwFBGREREpAEYyoiIiIg0AEMZERERkQZgKCMiIiLSAAxlRERERBqAoYyIiIhIAzCUEREREWkAhjIiIiIiDcBQRkRERKQBGMqIiIiINABDGREREZEGYCgjIiIi0gAMZUREREQagKGMiIiISAMwlBERERFpAIYyIiIiIg0geCjbunUr/Pz84O7ujqFDhyIyMvKN+2/cuBE9evSAh4cHOnXqhIULF6KgoKCKqiUiIiKqHIKGstDQUAQEBGDq1KkICgqCk5MTJk6ciLS0tFL3P3DgAJYuXYpp06YhNDQUP/30E0JDQ7Fs2bIqrpyIiIioYgkayjZs2IBhw4Zh8ODBaNKkCebNmwd9fX0EBgaWuv/Vq1fRokUL9OvXD/Xr10f79u3Rt2/ft/auEREREWk6HaGeuLCwEFFRUZg8ebJym7a2Ntq2bYurV6+Wep/mzZtj//79iIyMhIeHBxISEnDy5El88MEHaj+/VCotdbtEIlH7sd4Xr3vNb8L2UMX2eKE6twXA9ngV2+OF8rx3kCqxHR9lPWYEC2UZGRmQSqUwMzNT2W5mZob79++Xep9+/fohIyMDo0aNglwuR3FxMUaMGIEpU6ao/fw3btwosc3AwAAuLi5qP9b7IiYmBnl5eWXen+2hiu3xQnVvC4Dt8Sq2xwvqvneQKh4frydYKCuPCxcu4I8//sD3338PDw8PxMfH46effsLq1asxdepUtR7L3d29Wif10jg6OgpdgkZhe6hie6hie6hie7zAtqA3Ke34kEqlpXYGvUqwUFanTh1IJJISk/rT0tJgbm5e6n1WrFiB/v37Y+jQoQCev/Dc3Fx89913+Pjjj6GtXfYpchKJRHShTGyv923YHqrYHqrYHqrYHi+wLehN3uX4EGyiv56eHlxdXREREaHcJpPJEBERgebNm5d6n/z8/BLBS/Hi5XJ55RVLREREVMkEHb4cP3485syZAzc3N3h4eGDTpk3Iy8vDoEGDAACzZ8+GlZUVZs2aBQDo3LkzNmzYABcXF+Xw5YoVK9C5c2d+cyEiIqL3mqChrHfv3khPT8fKlSuRkpICZ2dnrF27Vjl8mZSUpNIz9vHHH0NLSwvLly9HcnIyTE1N0blzZ8yYMUOol0BERERUIQSf6O/v7w9/f/9Sb9uyZYvKzzo6Opg2bRqmTZtWFaURERERVRnBL7NEREREROUIZQkJCZVRBxEREZGoqR3KunXrhjFjxmDfvn28EDgRERFRBVE7lAUFBcHR0RGLFi1Cu3bt8N133/Hak0RERETvSO1Q5uzsjG+++QanT5/GwoUL8eTJE4waNQp9+/bFhg0bkJ6eXhl1EhEREVVr5Z7or6Ojg+7du2PlypX4/PPP8fDhQyxevBidOnXC7Nmz8eTJk4qsk4iIiKhaK/cpMW7cuIHAwECEhobCwMAAEyZMwJAhQ5CcnIxVq1bh//7v/7Bnz56KrJWIiIio2lI7lG3YsAF79+7FgwcP0LFjR2XvmOIkr7a2tli0aBH8/PwqvFgiIiKi6krtULZ9+3YMHjwYAwcOhKWlZan7mJqa4qeffnrn4oiIiIjEQu1Qdvjw4bfuo6enh4EDB5arICIiIiIxUnuif2BgIMLCwkpsDwsLQ1BQUIUURURERCQ2aoeyNWvWoE6dOiW2m5mZ4ffff6+QooiIiIjERu1Q9vjxY9SvX7/EdmtrayQlJVVIUURERERio3YoMzMzQ0xMTInt0dHRMDExqYiaiIiIiERH7Yn+ffr0wU8//QQjIyO0bt0aAHDx4kUsXLgQffr0qfACiYiIiMRA7VD26aef4tGjRxg3bhx0dJ7fXSaT4YMPPsCMGTMqvEAiIiIiMVA7lOnp6WH58uV48OABoqOjoa+vDwcHB9jY2FRGfURERESiUO7LLDVs2BANGzasyFqIiIiIREvtUCaVSrF3716cP38eaWlpkMlkKrdv3ry5woojIiIiEgu1Q9lPP/2EoKAgdOrUCU2bNoWWllZl1EVEREQkKmqHspCQECxfvhydOnWqjHqIiIiIREnt85Tp6urCzs6uMmohIiIiEi21Q9mECROwefNmyOXyyqiHiIiISJTUHr68cuUKLly4gFOnTqFp06bKc5UprFq1qsKKIyIiIhILtUNZrVq10K1bt8qohYiIiEi01A5lAQEBlVEHERERkaipPaeMiIiIiCpeuc7of/DgQYSFhSEpKQlFRUUqtwUFBVVIYURERERionZP2ebNm/Hll1/C3Nwct27dgru7O0xMTJCQkICOHTtWRo1ERERE1Z7aPWXbtm3D/Pnz0bdvX+zduxcfffQRbG1tsWLFCjx9+rQyaiQiIiKq9tTuKUtKSkLz5s0BAPr6+sjJyQEAfPDBBwgJCanY6oiIiIhEQu1QZm5uruwRq1evHq5duwYASExM5AlliYiIiMpJ7eHLNm3a4NixY3BxccHgwYMREBCAQ4cO4ebNmzx/GREREVE5qR3K5s+fD5lMBgAYPXo0TExMcPXqVfj5+WH48OEVXiARERGRGKgdyrS1taGt/WLUs0+fPujTp0+FFkVEREQkNmrPKevWrRt+/fVXPHjwoDLqISIiIhIltUPZ6NGjceLECfTu3RuDBw/Gpk2bkJKSUhm1EREREYmG2qFs3LhxCAwMRGhoKDp16oRt27bB19cXEyZMQHBwcCWUSERERFT9lfvalw0bNsQnn3yCQ4cOYevWrUhPT8eXX35ZkbURERERiUa5rn2pEBkZiQMHDiAsLAzZ2dno2bNnRdVFREREJCpqh7IHDx7gwIEDCAkJQWJiItq0aYPPP/8c3bp1g5GRUWXUSERERFTtqR3KevXqBXd3d4waNQp9+vSBubl5ZdRFREREJCpqh7KDBw+iQYMGlVAKERERkXipPdG/QYMGePbsGXbv3o2lS5ciMzMTABAVFYXk5OSKro+IiIhIFNTuKYuOjsa4ceNQq1YtPHr0CMOGDYOJiQkOHz6MpKQk/Pzzz5VRJxEREVG1pnZPWUBAAAYNGoTDhw9DT09Pub1Tp064fPmy2gVs3boVfn5+cHd3x9ChQxEZGfnG/Z89e4Z58+ahffv2cHNzQ48ePXDy5Em1n5eIiIhIk6jdU3bz5k3Mnz+/xHYrKyu1z+wfGhqKgIAAzJs3D82aNcOmTZswceJEHDx4EGZmZiX2LywsxPjx42FmZoYVK1bAysoKjx8/Rq1atdR9GUREREQaRe1Qpqenh+zs7BLb4+LiYGpqqtZjbdiwAcOGDcPgwYMBAPPmzcOJEycQGBiISZMmldg/MDAQT58+xY4dO6CrqwsAqF+/vrovgYiIiEjjqB3K/Pz8sHr1aixfvly57fHjx1iyZAm6d+9e5scpLCxEVFQUJk+erNymra2Ntm3b4urVq6Xe59ixY/D09MSPP/6Io0ePwtTUFH379sVHH30EiUSi1uuQSqWlblf3cd4nr3vNb8L2UMX2eKE6twXA9ngV2+OF8rx3kCqxHR9lPWbUDmVz587FJ598grZt26KgoABjxoxBamoqPD09MWPGjDI/TkZGBqRSaYlhSjMzM9y/f7/U+yQkJOD8+fPo168f1qxZg/j4eMybNw/FxcWYNm2aWq/jxo0bJbYZGBjAxcVFrcd5n8TExCAvL6/M+7M9VLE9XqjubQGwPV7F9nhB3fcOUsXj4/XUDmXGxsbYsGEDLl++jJiYGOTm5sLV1RVt27YtVwHqkMvlMDMzw/z58yGRSODm5obk5GSsW7dO7VDm7u5erZN6aRwdHYUuQaOwPVSxPVSxPVSxPV5gW9CblHZ8SKXSUjuDXlXua1+2atUKrVq1Ku/dUadOHUgkEqSlpalsT0tLe+1VAiwsLKCjo6MSpho1aoSUlBQUFhaqrAZ9G4lEIrpQJrbX+zZsD1VsD1VsD1VsjxfYFvQm73J8lCmUbd68ucwPOHbs2DLtp6enB1dXV0RERKBr164AAJlMhoiICPj7+5d6nxYtWuDvv/+GTCaDtvbzs3nExcXBwsJCrUBGREREpGnKFMo2btxYpgfT0tIqcygDgPHjx2POnDlwc3ODh4cHNm3ahLy8PAwaNAgAMHv2bFhZWWHWrFkAgJEjR+Kvv/7CTz/9BH9/fzx8+BB//PEHxowZU+bnJCIiItJEZQplx44dq5Qn7927N9LT07Fy5UqkpKTA2dkZa9euVQ5fJiUlKXvEAKBevXpYt24dAgIC0L9/f1hZWWHs2LH46KOPKqU+IiIioqpS7jllhYWFSExMhJ2dHXR0yv0w8Pf3f+1w5ZYtW0psa968OXbt2lXu5yMiIiLSRGpfZikvLw9fffUVPD090bdvXyQlJQEA5s+fjzVr1lR4gURERERioHYoW7p0KaKjo7F582bUqFFDud3HxwehoaEVWhwRERGRWKg97nj06FH88ssv8PT0VNnetGlTxMfHV1RdRERERKKidk9Zenp6qRcLz8vLg5aWVoUURURERCQ2aocyNzc3nDhxosT23bt3l+g9IyIiIqKyUXv4csaMGfjoo49w7949SKVSbN68GbGxsbh69WqpqyWJiIiI6O3U7ilr1aoV9u3bB6lUCgcHB5w9exampqbYsWMH3NzcKqNGIiIiomqvXCcYs7Ozw4IFC0psP3jwIHr27PnORRERERGJjVo9ZcXFxbhz5w4ePHigsj08PBz9+/fH559/XqHFEREREYlFmXvK7ty5gylTpihPFtulSxf88MMP+Oyzz3D37l0MHTqUJ48lIiIiKqcyh7IlS5bAzs4O3377Lf7++2+EhIQgNjYWQ4YMwdq1a6Gvr1+ZdRIRERFVa2UOZTdu3MD69evh7OyMVq1aISQkBJMnT8aAAQMqsTwiIiIicSjznLKMjAxYWloCAIyNjWFgYMDzkhERERFVkDL3lGlpaSEnJwc1atSAXC6HlpYW8vPzkZ2drbJfzZo1K7xIIiIiouquzKFMLpejR48eKj8PHDhQ5WctLS3cvn27YiskIiIiEoEyh7LNmzdXZh1EREREolbmUObl5VWZdRARERGJmtqXWSIiIiKiisdQRkRERKQBGMqIiIiINABDGREREZEGUCuUFRUVwcXFBXfu3KmseoiIiIhESa1Qpquri3r16kEmk1VWPURERESipPbw5ZQpU7Bs2TJkZmZWQjlERERE4lTm85QpbN26FQ8fPkSHDh1gbW0NQ0NDlduDgoIqrDgiIiIisVA7lHXt2rUy6iAiIiISNbVD2bRp0yqjDiIiIiJRK9cpMZ49e4bdu3dj6dKlyrllUVFRSE5OrsjaiIiIiERD7Z6y6OhojB8/HsbGxnj06BGGDRsGExMTHD58GElJSfj5558ro04iIiKiak3tnrJFixZh4MCBOHz4MPT09JTbO3XqhMuXL1docURERERioXYou3HjBkaMGFFiu5WVFVJSUiqkKCIiIiKxUTuU6enpITs7u8T2uLg4mJqaVkhRRERERGKjdijz8/PD6tWrUVRUpNz2+PFjLFmyBN27d6/Q4oiIiIjEQu1QNnfuXOTm5qJt27YoKCjAmDFj0L17dxgZGWHGjBmVUSMRERFRtaf26ktjY2Ns2LABly9fRkxMDHJzc+Hq6oq2bdtWRn1EREREoqB2KCsoKECNGjXQqlUrtGrVqjJqIiIiIhIdtUNZq1at4OHhgdatW8Pb2xvNmzeHvr5+ZdRGREREJBpqhzLF0OXFixexadMmFBcXw83NDV5eXvDy8kK7du0qo04iIiKiaq1cPWWtWrXClClTUFxcjBs3bmDnzp1Yu3Yt1qxZg9u3b1dGnURERETVmtqhDAAePHiAixcv4uLFi7hw4QKKiorg6+sLLy+viq6PiIiISBTUDmUdOnRAQUGBcrjyo48+gqOjI7S0tCqjPiIiIiJRUPs8ZaampsjLy0NqaqryX35+fmXURkRERCQaaveU7du3D8+ePcOlS5dw6dIlLFu2DLGxsXB2doa3tzdPIEtERERUDuWaU1arVi106dIFLVq0QPPmzXH06FGEhITg+vXrDGVERERE5aB2KDt8+LBygn9sbCxq166Nli1bYs6cOZzoT0RERFROaoey77//Hq1bt8bw4cPRunVrODo6vnMRW7duxbp165CSkgInJyd8++238PDweOv9QkJCMHPmTHTp0gX/+9//3rkOIiIiIqGoHcoiIiIqtIDQ0FAEBARg3rx5aNasGTZt2oSJEyfi4MGDMDMze+39EhMTsXjxYl7qiYiIiKoFtVdfRkVFISYmRvlzeHg4/u///g/Lli1DYWGh2gVs2LABw4YNw+DBg9GkSRPMmzcP+vr6CAwMfO19pFIpPv/8c0yfPh22trZqPycRERGRplG7p+y7777DpEmT4OjoiISEBMycORPdunXDwYMHkZeXh6+//rrMj1VYWIioqChMnjxZuU1bWxtt27bF1atXX3u/1atXw8zMDEOHDsWVK1fUfQkAnge70kgkknI93vvgda/5TdgeqtgeL1TntgDYHq9ie7xQnvcOUiW246Osx4zaoSwuLg7Ozs4AgLCwMLRu3RpLly7FlStXMHPmTLVCWUZGBqRSaYlhSjMzM9y/f7/U+1y+fBl79uxBcHCwuqWruHHjRoltBgYGcHFxeafH1WQxMTHIy8sr8/5sD1Vsjxeqe1sAbI9XsT1eUPe9g1Tx+Hg9tUOZXC6HTCYD8Hx+ma+vLwCgXr16yMjIKFcRZZWdnY3Zs2dj/vz5MDU1fafHcnd3r9ZJvTQVsSijOmF7qGJ7qGJ7qGJ7vMC2oDcp7fiQSqWldga9Su1Q5ubmht9++w0+Pj64dOkSfvjhBwDPJ96bm5ur9Vh16tSBRCJBWlqayva0tLRSHyshIQGPHj3Cxx9/rNymCIguLi44ePAg7OzsyvTcEolEdKFMbK/3bdgeqtgeqtgeqtgeL7At6E3e5fhQO5R99dVX+OKLLxAeHo4pU6bA3t4eAHDo0CE0b95crcfS09ODq6srIiIi0LVrVwDPQ1ZERAT8/f1L7N+oUSMcOHBAZdvy5cuRk5ODr7/+GnXr1lX35RARERFpBLVDmZOTU4lgBACzZ8+Gtrbaizkxfvx4zJkzB25ubvDw8MCmTZuQl5eHQYMGKR/XysoKs2bNQo0aNeDg4KBy/1q1agFAie1ERERE75NyXWYJeL5yMj09XTl8qGBtba3W4/Tu3Rvp6elYuXIlUlJS4OzsjLVr1yqHL5OSksoV9oiIiIjeJ2qHsgcPHuDrr78uccoKuVwOLS0t3L59W+0i/P39Sx2uBIAtW7a88b6LFi1S+/mIiIiINI3aoezLL7+Ejo4Ofv/9d1haWkJLS6sy6iIiIiISFbVDWXR0NAIDA9G4cePKqIeIiIhIlNSerNW4ceNKPx8ZERERkdioHco+//xzLFmyBBcuXEBGRgays7NV/hERERGR+tQevhw/fjwAYNy4cSrb32WiPxEREZHYqR3KNm/eXBl1EBEREYma2qHMy8vrtbfduXPnnYohIiIiEqtynzxWITs7GyEhIdi9ezeioqI4fElERERUDuUOZZcuXcKePXtw+PBhWFpaolu3bvjuu+8qsjYiIiIi0VArlKWkpCAoKAh79uxBdnY2evXqhcLCQqxevRpNmjSprBqJiIiIqr0yh7IpU6bg0qVL8PX1xVdffYUOHTpAIpFgx44dlVkfERERkSiUOZSdOnUKY8aMwciRI9GgQYNKLImIiIhIfMp88tht27YhJycHgwYNwtChQ/HXX38hPT29MmsjIiIiEo0yhzJPT08sWLAAZ86cwfDhwxESEoKOHTtCJpPh7NmzPJs/ERER0TtQ+zJLhoaGGDJkCLZv3479+/dj/Pjx+PPPP9G2bVtMmTKlMmokIiIiqvbUDmUva9SoEWbPno2TJ09i2bJlFVUTERERkei888ljAUAikaBr167o2rVrRTwcERERkei8U08ZEREREVUMhjIiIiIiDcBQRkRERKQBGMqIiIiINABDGREREZEGYCgjIiIi0gAMZUREREQagKGMiIiISAMwlBERERFpAIYyIiIiIg3AUEZERESkARjKiIiIiDQAQxkRERGRBmAoIyIiItIADGVEREREGoChjIiIiEgDMJQRERERaQCGMiIiIiINwFBGREREpAEYyoiIiIg0AEMZERERkQZgKCMiIiLSAAxlRERERBqAoYyIiIhIAzCUEREREWkAhjIiIiIiDcBQRkRERKQBGMqIiIiINIBGhLKtW7fCz88P7u7uGDp0KCIjI1+7765duzBq1Ci0bt0arVu3xrhx4964PxEREdH7QPBQFhoaioCAAEydOhVBQUFwcnLCxIkTkZaWVur+Fy5cQJ8+fbB582bs2LED9erVw4QJE5CcnFzFlRMRERFVHMFD2YYNGzBs2DAMHjwYTZo0wbx586Cvr4/AwMBS91+6dClGjx4NZ2dnNG7cGAsWLIBMJkNEREQVV05ERERUcXSEfPLCwkJERUVh8uTJym3a2tpo27Ytrl69WqbHyMvLQ3FxMWrXrq3Wc0ul0lK3SyQStR7nffK61/wmbA9VbI8XqnNbAGyPV7E9XijPewepEtvxUdZjRtBQlpGRAalUCjMzM5XtZmZmuH//fpkeY8mSJbC0tETbtm3Veu4bN26U2GZgYAAXFxe1Hud9EhMTg7y8vDLvz/ZQxfZ4obq3BcD2eBXb4wV13ztIFY+P1xM0lL2rNWvWIDQ0FJs3b0aNGjXUuq+7u3u1TuqlcXR0FLoEjcL2UMX2UMX2UMX2eIFtQW9S2vEhlUpL7Qx6laChrE6dOpBIJCUm9aelpcHc3PyN9123bh3WrFmDDRs2wMnJSe3nlkgkogtlYnu9b8P2UMX2UMX2UMX2eIFtQW/yLseHoBP99fT04OrqqjJJXzFpv3nz5q+9359//on//e9/WLt2Ldzd3auiVCIiIqJKJfjw5fjx4zFnzhy4ubnBw8MDmzZtQl5eHgYNGgQAmD17NqysrDBr1iwAz4csV65ciaVLl8LGxgYpKSkAAENDQxgZGQn2OoiIiIjeheChrHfv3khPT8fKlSuRkpICZ2dnrF27Vjl8mZSUBG3tFx16O3bsQFFRET755BOVx5k2bRqmT59epbUTERERVRTBQxkA+Pv7w9/fv9TbtmzZovLzsWPHqqIkIiIioiol+MljiYiIiIihjIiIiEgjMJQRERERaQCGMiIiIiINwFBGREREpAEYyoiIiIg0AEMZERERkQZgKCMiIiLSAAxlRERERBqAoYyIiIhIAzCUEREREWkAhjIiIiIiDcBQRkRERKQBGMqIiIiINABDGREREZEGYCgjIiIi0gAMZUREREQagKGMiIiISAMwlBERERFpAIYyIiIiIg3AUEZERESkARjKiIiIiDQAQxkRERGRBmAoIyIiItIADGVEREREGoChjIiIiEgDMJQRERERaQCGMiIiIiINwFBGREREpAEYyoiIiIg0AEMZERERkQZgKCMiIiLSAAxlRERERBqAoYyIiIhIAzCUEREREWkAhjIiIiIiDcBQRkRERKQBGMqIiIiINABDGREREZEGYCgjIiIi0gAMZUREREQagKGMiIiISAMwlBERERFpAIYyIiIiIg3AUEZERESkATQilG3duhV+fn5wd3fH0KFDERkZ+cb9w8LC0LNnT7i7u6Nfv344efJkFVVKREREVDkED2WhoaEICAjA1KlTERQUBCcnJ0ycOBFpaWml7v/PP/9g1qxZGDJkCIKDg9GlSxdMnToVd+7cqeLKiYiIiCqO4KFsw4YNGDZsGAYPHowmTZpg3rx50NfXR2BgYKn7b968GR06dMB//vMfNG7cGJ999hlcXFzw119/VXHlRERERBVHR8gnLywsRFRUFCZPnqzcpq2tjbZt2+Lq1aul3ufatWsYN26cyrb27dsjPDy8TM8pl8uVzy2RSErcLpFI0KBpPejqlbztfWVjbwmpVAqpVKr2fSUSCRraW0JXR/D8XmHq25i/U3s0trGoVu1hZ2VWrvaQSCRoamUBPUn1aQsAsDcvf3s4mFlAT7t6tUeDOuVvj4a1rKCrVX3eS+sbl/+9g1RJJBLU0m8ILbmu0KVUGGP9+q89PhTbFBnkdQQNZRkZGZBKpTAzM1PZbmZmhvv375d6n9TUVJibm5fYPzU1tUzPKZPJAAC3bt167T6d+jkDcC7T470vrl27Vu77+razA2BXYbVogndpjx4etgBsK6wWTVDe9ujX2A5oXL2ODaD87THAogFg0aAiS9EI5W0PP73GgFnjii1GYO/y3kGqdNENpkIXUZGy3n58KDLI6wgayoSgo6MDd3d3aGtrQ0tLS+hyiIiIqJqTy+WQyWTQ0Xlz7BI0lNWpUwcSiaTEpP60tLQSvWEK5ubmJXrF3rT/q7S1taGnp1e+gomIiIgqiaCTH/T09ODq6oqIiAjlNplMhoiICDRv3rzU+3h6euL8+fMq286dOwdPT8/KLJWIiIioUgk+I3X8+PHYtWsXgoKCEBsbix9++AF5eXkYNGgQAGD27NlYunSpcv+xY8fi9OnTWL9+PWJjY/Hrr7/i5s2b8Pf3F+olEBEREb0zweeU9e7dG+np6Vi5ciVSUlLg7OyMtWvXKocjk5KSoP3SaqYWLVpgyZIlWL58OZYtW4YGDRpg9erVcHBwEOolEBEREb0zLfnb1mcSERERUaUTfPiSiIiIiBjKiIiIiDQCQxkRERGRBmAoIyIiItIADGVEREREGoChrAoVFxcLXQIREVG19j6fVIKhrAooDpDQ0FBkZmYKW4yGkEqlyM/PF7oMjZGQkIB9+/YJXYbgFH8r6enpiImJEbga4SnaIycnR+BKNEtubq7QJZCGUVzou6CgAFpaWu9tMGMoqwJaWlooKirCrVu3YGJiInQ5glL8oVy/fh1Hjx4VuBrNkZiYCENDQ6HLEJyWlhYAIDIyEsePHwfw4s1WjBTtMX78eKSnpwtcjebYtWuX6L/UKd5LT58+jT///FPgaoSn+FuZNWsWzp07p/z5fcOTx1YBuVz+3h4gleXMmTMwNjZGs2bNRN8+MpkM2traom+HlxUWFkImk0FfX1/oUjRCamqq8ionYpeVlYXt27dj0qRJQpeiEWJjY/HkyRP4+PhAKpVCIpEIXZKg0tPTUbt27fe2HRjKKpniA/f+/fuwtbWFrq6u0CWRhikuLkZaWhqsrKyELkVjKP5uCMoPWrYJv+DSm1WHv5H3u/r3gOIA2bp1K1JTUwWuRlgvzxeKjo4WuBrNcf36dfzvf/8TugzBKY6PmJgYbN++XeBqhKdoj5CQkBLXABYjRSA7f/488vLyhC5HI+Tk5CAiIkLoMgSnmOJw8eJF3LlzR+Bq3o24/8qriEwmw9ChQ1GvXj2hSxHUy/OFTpw4AUDc84UUXFxc8OmnnwpdhuAUx4dUKlUO1Yn5+FC0R3x8PMzMzASuRniK9ggPDxd9QFWIiorC/fv3AbzfKw7fleJ4+Oeff977Y4PDl1TlxDxfiMMvVFY8VkoqLi5GUlISbG1thS5FI+Tk5CA/Px9mZmY8XqqJ9ztSaihFzpVKpQCA7du3IykpSciSqtybsr6Ojo4oAxmAEm+aGRkZ2LNnj0DVaJ7CwkKcPHlS6DIEJ5PJoKWlhZSUFADi7gV5mY6ODgPZS4yMjJS9qGINZIq/jbS0NGRnZwtczbtjKKsEij8OxeqP+Ph41KhRQ8iSqoziD+TloSiZTKYchhLzfKGIiAgkJCSobKsubyQV5f79+7h37x4AcQcRxRDMkiVLEB0dLdoPXMUXW8WxsHDhQjx58kTIkgSneC+9f/8+fvjhB2GLEUhp7w2hoaE4e/asANVULIayCvDyAfLw4UOcOHEC//77r3LbnDlzYGpqKkRpVU5LSwuHDh3Cw4cPATwPptra2soPmcLCQtHNF8rKygIAbNy4UdlDqPiwqVOnDsaNGydUaYKTy+XIzMxUTtxu1KgRBgwYIGxRGmT69OlwcnISugzBKL7YamlpQSaTwcbGBpaWlgJXVXUuXbqE/Px8ZGdnK98vFe+lRkZG6N27NwDxvJcqaGlp4c6dO8jPz1d+YenQoQPat28vcGXvTkfoAqqD4uJi6OrqYvny5bhz5w7kcjmys7NRq1Yt1K1bF8OHD4eDg4PQZVYJqVSKXbt2IT4+HoaGhvD29kbLli3h7e0NExMTuLu7w93dHQDe+wmZZZGfn4/jx4/j4sWLuHv3LhISEqClpaUMpvPnz8f8+fNhbGwscKVVq7i4GD///DMiIiLg5uaGRo0aoUmTJqhfvz6aNm0KQLzDMS+rX7++0CVUKcUpDdLT07F7925cv34dU6ZMgYeHB7S1tfHhhx9Wi9MelNWJEyfQunVrrFy5EkZGRrC3t0f9+vVRr149WFlZKU+jI4b2UPzeY2NjcfToUZw8eRKNGzfGjz/+iKSkJFhbW0NPT0/oMt8ZQ1kFUJx77OjRo5gzZw5sbGyQlpaGBw8e4MqVK6K65qVEIsG6desQHR2NdevW4eTJk4iMjMTixYthY2ODFi1aYMaMGUKXWWVkMhksLS2RlJQEqVSK4OBgFBcXw8rKCnl5eYiLixNVIFO8sZ44cQK3b9/Gt99+ixs3buDWrVs4fvw46tati2XLlgldpkYQ88TtL7/8El26dMGVK1eQkZEB4HlPc+fOnWFvby9wdVVnwoQJKCoqQl5eHpKSknDjxg0YGBjAzMwMNjY2GDFihGiOEcWI1M6dO2Fubg47Oztlr+mpU6dgZGSEvn37CllihWAoe0fJycl4+PAhzMzM4OLiouw+bdiwITw8PODn5yeq5eyKE10ePXoUNjY2mDFjBkxNTXH48GFs3LhRZRHE+3rGZXUYGhqiTZs2qF27NtLT06Gjo4Pbt28jOTkZOjo6+L//+z+hS6xSim/0ycnJ6NmzJ7y8vODl5QUAyMzMVA77i6k3pLCwEOHh4ejZs6fKaxbLh+3LtLW1UVBQgLS0NAwbNgx79+5Fs2bNAAD79++Hn5+fwBVWLcVnx5w5c1BUVITY2FhER0fjxo0bePLkifIaj2I4VhR/G5GRkdixYwdmzZqFli1bAgBOnjyJHj16CFlehWEoe0c3b97EunXrYG5ujgcPHmDhwoUYOHAgmjZtCj09PVEFMuDFB8nFixcxa9YsWFtbAwD69++PW7duwdvbG4A4uttf5uzsrPy/t7c3cnJyoKurWy2628ujY8eO2LhxI86ePYvGjRvD1NQUJiYmMDExgVwuF8XxkZ+fD319fdy4cQMnTpxA7969lR+wOTk5yMzMhI2NjdBlVrmoqCg4OTkhIyMD+vr6MDExwePHjwEAdnZ2AldX9RRfZHV1deHk5AQnJycMGDAAhYWFAldWtRR/F7a2trhy5QpiY2Ph4+MDAHj06FG1mE8GMJS9M3d3d8ycORNJSUm4e/cu7ty5g5UrV0JXVxc1a9bEhx9+CEdHR6HLrDKKazi2aNECq1atwscffwx7e3uYmprizJkzGDJkiNAlCublibpGRkYCVyOczMxMzJ8/Hzdv3sSTJ09gY2MDOzs71K1bF506dRJFDyrw/JqF+/btw7lz5+Dg4ICUlBQYGxtDX18fFy9exNWrVzFz5kyhy6xyDg4OMDAwwOjRo+Hu7o7CwkLs379fOS9XLL3sCi9/SZHL5cqfFV/oxNBLBjx/7UZGRhg4cCD++OMPGBsb48KFC9i3bx8aNGhQbTpAGMrekaWlJSwtLXHy5ElMmjQJaWlpePz4MR49eoSYmBhRfvhqaWlh3Lhx+PXXXxEUFITi4mJERUWhRYsWaNKkiXIfsRFD78+bKIYkjx49Cn19fRw5cgQRERG4desWzp07BzMzM1ENT5mbm8Pb2xu7du2CtbU1fvjhB0gkEtja2uLs2bOYOHGi0CUKombNmhg5ciTy8/Px7Nkz9OzZE35+fpgyZQoA8b13KN43ZDKZaIYqS6N43W5ubujUqRNCQkKwfv16tGnTBt27dxe4uorDM/q/g8LCQujo6CAiIgIbNmzA2rVrlbcVFBTg9u3b8PT0FK5AgSUnJyMiIgJSqRRNmzZF48aNYWRkJMo3loKCAqSmpsLKygo6OuL8LqQIZb///jvq1q1b4tQXaWlpMDMzE9V8MplMhsjISJiZmSE+Ph4JCQnK8xpOnDgRNWvWFLrEKldcXAwdHR1ER0cjLS0NTk5OqFmzpmjO9Qi86A08fvw45HI5fHx8YGBgoLKPGN9HX/Xs2TPUqlVL6DIqlDg/HSrI/fv3cfbsWZw+fRqWlpbIzMxEQUEBrKyscP78eVy/fl3UoczKyqrUc06J7Y1kx44dCAsLg5GREZYsWaL80BFbOFMErdTUVBw+fBjFxcVwcXFB3bp1YWpqqrxUjBgCmeJD9/Lly0hLS4Onp6eoz1SvCOIJCQnYtm0bQkJC4OPjAzc3NzRr1kxUgQx4cX62c+fOYceOHcoRmY4dO6JDhw5wc3MT3fuoguJk5Do6OtUukAE8eew7MTU1hZ6eHh48eIA7d+7gm2++wdq1axEcHIzff/+9Wh4w6lDMfxAjxeu+ePEiTpw4gcGDB+Pff/+FoaEh7t69i6VLlwpcoTBycnJgaGgIR0dHnD59Glu2bMH69euxbds2UX7zDwwMVF7RYfXq1WjWrBmWLVuGgoICgSurWoq/l0WLFkFPTw8rVqxA69atERwcjO+//x5FRUUCV1j18vPzUVBQgFmzZmHx4sXo06cPdu7cidmzZ2P27NnKk1KLjba2drX+Qlt9X1kVsLS0xJgxY1CvXj20aNECV69exeXLlxEeHo4ePXpg0KBBQpdYZRQfqKmpqahVqxb09PQgk8lENSH3ZYrXfv78eXTt2hW1a9eGi4sLgOc9rPHx8QJXKAwjIyPMnDkTxcXFuHfvnnJ5f0pKiqiW9yv+LmJiYrB48WKcO3cODx8+xJ9//on169cjKSkJDRo0ELbIKvTy+8SoUaNgZWWF5s2bY8iQIRgxYgRiYmLg5uYmYIVV5+Vz+T158gQ//vgjiouL0aJFCxgaGiIuLg6ZmZnYtWuX6OYdimEaSPV8VVVE8cfTtWtXAECXLl3QpUsXAOJbIaSwd+9eFBYWYtq0aaJ8/QqK156XlwdnZ2f89ddfypWnV65cQbt27YQsTzCKHhEdHR1RL+8Hns+51NXVxZ49e3Dw4EGMGzcOXl5emDdvnvJUMmKgCOKPHz+GXC5HZGQkunXrBuD5h3B2draoAqriS0lubi709PSQlZWFmjVrKrdraWmhR48e2L17t5BlVjmxTAOpXq+mimlrayuDmWKoTi6XQyKRiC6QKN4wvLy8sGjRIly+fBmffvopmjdvLnBlwho1ahRmzZqFmzdvwsfHBw8fPsTdu3cxefJkoUsTBJf3PyeTyWBlZQV/f38cPXoUTZs2Rfv27XHmzBmYmZmJ6vx1it95XFwcIiIicOrUKbi7u8PNzQ0FBQUYMWKEqBY8KNrD19cXx48fx08//YSWLVsiISEBly9fxqxZsxAREaE8cWp1pgjsL08D2bhxIwwNDXHjxg2EhoZizpw5QpdZobj6sgKJZejlbQoLC7Fx40akpaVh0KBByvO0iaV9FEH9+vXryMjIgJ2dHfbs2YOLFy+idu3amDNnjmiuhfo6Ly/vF8PE/pcpXrfim75CUVERdu7cCUNDQ1FNfXiZXC5HdHQ0zpw5g1OnTuHBgwfQ1tZGkyZN8Mknn4hu4VRGRgZ27Nih7EVs06YNunXrhunTp+Pbb7+t9otDFCNOK1euhLW1NSwsLHDkyBEsWLAA+/btw+HDh7F69Wqhy6xQDGXvIDc3F0+ePBFV1/rb5OTkwMjICImJifj0009x//59fPvtt6L6kFGEssWLF6NevXoYO3Ys8vLyVJa0iyWgAlze/zo9evSAjY0N2rRpg3bt2sHV1RWAeKc+5ObmIjk5Gdra2rCwsIChoSGys7MRFRWFkJAQDB48WHnJJTFITEzEs2fP0LRpU+X1lRWSk5OVFyMXg8WLF6Nv377KaSAtW7bEd999BycnJ4waNUro8ioUhy/LQfGhGxYWhoiICCxZsgTR0dH4+++/4eXlhY4dOwpdYpVStMfWrVsRGxuLs2fPws3NDZ6enrCwsMCmTZvw8OFDfPzxx9DX1xe63Eqn6PnJycmBiYkJAIg2kAFc3l8amUyG//73v4iMjMSlS5cQGBgIAGjZsiUWLlwocHVVR/HeERcXh/Xr1yMsLEx5dvYBAwagZ8+e8Pb2Vl6erbpTBPL//e9/iI6ORk5ODgoKCmBrawtbW1sMGDAA1tbWsLKyEtX7iJimgTCUlYOic/HUqVMYPHgwnj59iq1bt+LOnTuIjIxEnTp14O7uLnCVVUcRQuLi4tC8eXOMHj0aGRkZcHNzg76+Pp4+fYpx48ahd+/eornkVHZ2Nm7evInAwECEhISgQ4cO8PHxQePGjUXzRvqyl5f3u7m5ITo6GmvXrsWBAwfg5uaGb7/9FsbGxkKXWSUU8+g8PDzg7u6OkSNH4urVq9i8ebPyihdioXgv3bZtG7S1tXHp0iX8+++/CA4Oxl9//QVXV9dqP0T3MsUXmAMHDuDzzz+HjY0NUlNTce/ePVy+fBl+fn6wtrYWxQmWX50GsmjRIuzZswebN29G7dq1MW/evGp5bViGsnJQ/OEkJiaiYcOGCAwMhI2NDebPn4+PP/4Yubm5AlcojK+//hpFRUUlutpr164Na2trNGzYUKDKqp6uri6WLl2KxMREXLt2DRcuXMBff/0FZ2dn/PLLL0KXV2W4vF+VoncjJSUFFhYW0NLSgkQiQatWrRAeHi66UKYIFgYGBujRowcAoG7dupgyZQoiIyNx/vx52NraiiKEKI6NhIQEeHp6KlfyA0CLFi3Qq1cv5ZBldW+Llx08eBD16tWDr68vpk+fXu1HHRjKyqmwsBC+vr744osvkJGRge3btyM3NxePHz9Gq1athC6vyii6269evYoLFy7g8ePHqFmzJhwdHZX/iouL8c0334hmRZlcLkeNGjVga2uLhg0bokOHDpBKpYiLixPdSTC5vF+VlpYWCgoKMH78eNSuXRuurq5o06YNXF1dcfz4cQwfPlzoEquUlpYW8vLycPz4ceWVDSQSCXJzcyGVSkUzbAm8CBgxMTG4cuUK1qxZg969e6N+/fowNDSEoaGh0CVWKbFOA+FEfzUpJrIrXL16FbVq1ULjxo0RHh6OoKCgarca5E0U32AVJ9F1dHREVlYWUlNT8fjxY0yaNAlt2rQRuswqd/nyZVy8eBEZGRmwsbFB69at0aBBA1FeoB4A0tPT8f3338PIyKjU5f21a9fGmDFjhC6zShQWFiI6OhqxsbG4du0aoqKi8O+//2LIkCH47LPPhC6vyqWkpGD+/Pm4ceMG0tLSYGtrCwMDA3h6euKbb74Rurwqd+LECYSEhODJkyfQ19eHsbExLCwsMGrUKFEN5QLPp4GMHTsWMTExaN++vco0kOqKoUxNkyZNwsKFC/H333/D1tYWHTp0UPYApaWlITs7G/b29gJXWbXy8vIwadIkbNmyBcDzD+Dk5GTExcWhQ4cOqFmzZrX9VvMyxWu8c+cO5s2bBycnJ5w5cwYeHh6IjIxEgwYN8Pvvv1f7dngdsS/vf9mrfw+FhYXKHlYxi46OxunTp3HlyhXcu3cPycnJmDRpEqZPny50aVVGMfqQnp6O6Oho3L17F7dv38b06dOr5RyqNykoKMDjx4+V00Du3LmDu3fvVutpIAxlasrMzISJiQm+++47nD9/HpmZmbCxsYGvry98fX1FtWRb8cFy+/ZtLF++HJ9//jmaNm0qdFmCUbyZrlmzBgYGBmjSpAl2796NL774AgsWLEDHjh1FNzylwOX9L5w6dQrh4eHIyMiAp6encvhSzKKiomBlZQVzc3PlttzcXFy7dg01a9aEh4eHgNVVHZlMhtjYWAQHB6NBgwbo0qULTE1NRfc3ApR+Pr+Xp4E4OTkJXGHlYCh7B3K5HPHx8bh48SKOHTuGuLg4hIWFCV1WlVH80YSGhuK3336DoaEh2rRpg/r168Pe3h5ubm6imgehaI9JkyZh2rRpCAkJgbOzMwYMGIBffvkF5ubmohmiA8q+vB+ovvNDXpWUlIRhw4Zh0qRJKC4uxvXr1xEfH4+8vDzs3bu3xPnbqrvMzEysX78ecXFxuHfvHkxNTeHl5QUPDw/4+voKXV6VUfytbN68GRERETAxMcHdu3fx6NEjzJ49GwMHDhTN38jLxDgNhBP934GWlhbs7e1hb2+PoUOHori4WOiSqpTiDcLd3R3ffvstHjx4gIcPH+LatWs4cuQIPv30U1H1ACjaY8CAAbC1tYWFhQWOHTuGmjVr4tixY1i8eLHAFVYtLu9/QfGBev36dXTr1g1jxoxBbm4uBg0ahPT0dKSlpYkqkCl+50eOHEFqaipatGiBvLw8eHl5KUOar6+vaIKI4vg/fPgwpk2bppyHe+bMGezcuRMtW7aEnZ2dkCVWmZengfzyyy8q00C2b99e7aeBMJRVoOp2YdSyUvR8eHl5obi4GElJSbh3755ohzJ79+4NABg+fDgeP36MAwcOwNvbGy4uLgJXVnW4vF+V4gPE3t4eDx48AABlL3Lt2rVFdboY4MX5yc6fP4/x48fj9OnT8Pb2xn/+8x+kpKSgXbt2yv2q64fvy7S0tJCTk4OcnByVhVHt27fHsmXLRDXXUCaTQSKR4MSJE+jZsyeaNGmCjIwMzJw5UzkNpDofE+JMEfTOFN90Hz16hPDwcCQkJMDMzAxNmjRBkyZN4O3tLZpTYLyOsbExPv74YxQVFSmH6cSCy/tLd/PmTWzfvh3bt2+Hj48POnTogDZt2qjMpRIDRS9qVlYWLCwscO/ePQwdOhQAcO/ePXTu3BmAeC5QL5VKoaOjg5YtW8Lf3x+TJ0+Gvb090tLSUFxcLKr5ZIovaZcvX1ZOA+nYsSPq1auHJk2aoLCwUOAKKxfnlKnp+PHjePDgASZMmCB0KYJSzIGYO3cu6tati5CQENStWxfp6enQ0dHBzJkz0alTJ6HLJIFxef+LgHrr1i1MmjQJy5cvR3Z2Ni5fvox//vkHycnJOHLkiCh6DF/16NEjmJqa4uDBg1i1ahVcXV3x4MEDBAcHi/L6n2lpaVi/fj3S09Nx+/ZtGBsbY+LEifD19RXdNVFDQ0Ph4+ODwMBAREZGon///lixYgUWL15crUcd2FNWRg8fPoSlpSWOHz+uXAmkWBVy8eJF1KxZs1ofKK9SvDnExMRg0aJFiIyMxA8//IAnT55g+fLlqF+/PgDxDD9Q6Tp06ABfX98Sy/vFGEBSUlIwYcIEtGrVCkVFRejYsSPy8vKQnZ0tyvYAoDzFw8CBA6Gvr49Hjx5h5MiRkEgkonnvmDt3Ltzd3eHj44NGjRrhiy++QGxsLAwMDGBoaKg8caqYAhkg3mkgDGVldPXqVcybNw/5+fl49OgRDA0N0bp1a1hYWGDlypUYOXJktT9YXnXv3j3Url0b2dnZSEtLg52dHezs7FCjRg00aNAAgHiGHwD2or5KJpPh/v37Ksv727ZtK7rl/Yo5MkeOHMGzZ8/w77//om7dugAAIyOjar2SrDSKsBUXF4egoCDo6uqie/fu6NWrV4l9qrvs7GxYWFjgwoULCAwMhJ6eHtzc3NCuXTu4u7srA5mYiW0aCIcv1fD06VPMmjULDRo0wLFjx/D48WPY29vD2toaq1atEt2ba2pqKh48eICGDRti0aJF8PDwwNOnT5XXeRTDqjrgRS9qQEAAPDw8MGTIEFH3onJ5f+kCAgJw7NgxPHnyBI0aNUKHDh3QsWNHUV2WTaG4uBgDBw6Et7c37t27h2vXrkEikcDOzg7r169HnTp1hC6xyhQWFiInJwcpKSm4d+8eIiMjcffuXSQkJMDLywsLFiwQukSqQgxlZaS40HZBQYFyJUxBQQEuXrwIS0tLODo6ClyhsE6cOIE1a9bAwcEBXbp0UV7vUQxd7sHBwcpe1LZt22Lw4MHKXlR/f3+MHDkSffr0EbrMKqMIXP7+/qUu7//iiy9Es7y/NA8fPsTFixdx/PhxxMXFITQ0VOiSqlxUVBT++9//YuPGjcptt27dwunTpzF58mThChOYXC5HYWEhnj17hvj4eOjp6cHd3V0076XEUFYmig+ZgoIChIaGIjc3F97e3mjSpInQpQkqKysLZ86cgZGREVq1agU9PT3RnhaEvaiqcnJy4O/vj6CgIJXtgwYNwm+//Saq4cs3efls5WKwadMmGBoawsjICCkpKfjwww+FLkkjiGVUoSzEPg1EPO8G70AxzDJ9+nQ4ODhgy5YtMDAwgLGxMWxtbfHf//4XZmZmAldZNRRvHtevX8fChQthY2ODoqIibNq0Cebm5nBzcxPVWeuB572otWvXxurVq1GjRg188803Kr2oYgtkXN5fdmIKZACQn5+PPXv2ICMjA5mZmbh8+TKGDBmCNm3aiOpcXK9iIONiOgVxvSOUg6KX7ObNm5DJZBg+fDj++ecfrFy5EnPnzkWNGjVEE8iAFyd9DA8Ph5eXF8aNG4ekpCT8+++/KqvqxPLNTy6XK4e1X+1F7dChg9DlCUIikUAikeDjjz/G+vXrERoaqlzeP3PmTADgcIxITZ48GZMnT0ZKSgquXbuGY8eOISAgABkZGTAxMUFgYCBq1qwpdJlVQvEeeefOHRgZGcHGxgYymQyAOEMaF9M9x+HLt1D84SguMm1vb499+/Zh6dKlCAsLQ3x8vCjnQKxYsaLEBdizsrKgra0tup4hAJg0aZLoe1GBksv7AZS6vF9MxD4cUxZxcXG4efMm+vbtK3QpVW7NmjVISUnBuHHjlKcIEStOA2FP2VspvrHo6+vDzc0Njx8/hrGxMYDnlwixsLAQsjxByOVydOrUCd988w369++Ptm3bwtXVVdkuYsFeVFVc3q+KwzFl16BBA+VpdMSmZ8+eCAwMxOjRo9GyZUt8+OGHcHNzE11vGaeBPMeeMjUUFhYiIyMDn376Ke7duwcXFxf89NNPojkzuaLXUHH2bTc3N2hpaeHff/9Ffn4+WrZsic8//1zoMqsMe1FL4vL+F7gql17n5cVjOTk5qFWrFrKysrBixQrcuXMHI0aMQP/+/YUus8pwMd0L7ClTg56eHqysrLBjxw7cvXsXJiYmouwpu379OkaOHInRo0fj8ePHyMjIwL1795TfZMQyX4i9qCXp6elBT08PderUQdOmTdGlSxeV5f2AeI6PAQMGoHPnzsrhmCVLlqgMx/j6+gpdYpVRfOg+ffoUSUlJcHJyErokQSlOKPzbb7/h0KFD0NHRgbGxMezs7JCYmIgTJ06gf//+ovlb4WK6FxjK3kDxB3HhwgWsX78esbGxaNeuHXx9feHh4QEzMzNRnQRTEUIKCwuV52WztraGtbU1nJyclIsAxPAm8rKxY8eisLAQ1tbW2LJlC1q1agUXFxf85z//Ebo0QSh6ELW0tFCjRg1YWFioBFSxHB8cjnnh5asaXLlyBQEBASgsLISenh4KCgqgo6MjmuMCePE3YGNjg7p166J3797w8vKCvr4+fvzxR+VnihiGMDkNRBWHL99AMf+jV69emDx5MgwMDHDq1Clcu3YNsbGx+OOPP0R30e3MzEyMGDECycnJ6NmzJ9q3bw8vLy9R9gq9jph7Uek5DseoOnjwIMLCwhAXF4ehQ4fC399fedvKlSthYmKCsWPHClihcB4/foz169fjyZMn6NatG9q2bSuqEMJpIKrYU/YGinMI2dvbY8CAAQCAHj16AACSk5NFN3EZeD5Ut2zZMjx48AA3b95EUFAQVqxYIbr5QuxFVcXl/ao4HKPK29sb8fHxOHfuHAICAvDrr7+iVatW6NWrF86cOYMZM2YAEMepdBSv8ebNmzh06BCaN28Ob29vBAYGYu7cuRg5ciS++eYbocusMpwGooqh7DW2bt2KP/74Az179oS5uTkuXLgAb29v5e1iPQGmvr4+XFxc4OLigq5du4p2vpCig/mHH35Q6UVdsmSJKHtRFW+sJ06cEP3yfg7HlFSnTh1MmjQJNjY26NOnDyIiInDw4EGsW7cO3bt3h4+PDwBxBHjFa0xNTcWjR48QFRUFGxsbeHl5oXbt2rC0tAQgvqs9cBrIcxy+fI3s7GwEBQXhxIkTuH79OvLy8tCmTRvldR3FsuLyVWL4JquOKVOm4Pfff1fZpuhFFeMZyuPj4xEYGIh9+/aJdnk/h2NKUnxZy8nJKTGXTkw9ym+iuL4yPSfWaSAMZa+xe/duDB06VPnzrVu3cPDgQZw6dQrR0dHYsGGD8tsdicvLvai5ubno16+fSi+q2HB5f+k2b94Md3d3PH78GJcuXcIPP/yA77//HhYWFpg2bZrQ5VUZRUgNCQnB+fPncf78edStWxe+vr5o3749HB0dRfdlr7CwEKdOncKRI0dgbGwMe3t7DBo0SFSLPwBOAykNQ1kpkpOTMXv2bOjr62PAgAHo1q1biW5kMR0onC+kir2oqhRvrMuXLy+xvP/cuXNo1aoVli1bJpqh7ZeJ/dyGCoWFhRg4cCC+//57fPfdd+jZsyd27NiBzMxMHDp0CPb29kKXWCUUfwO7du3C8ePH0aBBA1hbW+P06dOoXbs2fvzxRxgYGAhdZpXhYrqSGMpe48GDB9i9ezeOHz8ONzc39OvXD+3btxdlCFHg5UCeYy9q6Xbv3o3Q0FCV5f116tSBlpYWdHV1RfVFpjRiHI5RfKE7c+YMDhw4gC+++ALTpk3Djh07sGPHDhQVFWHMmDFCl1llFO3x8ccfY/z48fDy8gLwfFX7t99+i1GjRonyvYPTQF4Qb8J4i4YNG2L27NnYsmULrK2tsXXrVvz+++949OiR0KUJpmfPnjA0NMTo0aMxa9YsREZGKnvMxCI5ORl///03Jk+ejLCwMBQXF8PFxQUzZ85EcHAwoqOj0aZNG6HLFMTQoUPx008/ISYmBkuXLsXFixeRlZWlnCcjhkAmlUoBABcuXMDkyZPRtWtXfP/99zh+/DhMTU1hYWEBMX0PVvzOY2Ji4O3tjX/++QcODg4Ank/+j4yMBADRvI8ovtQXFxcjJSUFwPNRFxMTE2RkZCgXTInhGNm6dSs6duyIhQsXKhfTvczKykp0gQxgT1kJUqkU165dQ3JyMvLy8uDm5ob8/HwEBgZi165dMDY2RnBwsGh6ijhfqCT2or5Q2vJ+qVSKwMBAnD59WnTL+zkcU7o7d+5AX18f+fn5mDt3LhwdHZGYmIgBAwZg8ODBohvaPnXqFNauXYsPPvgArq6uOH/+PEJDQ7Fr1y6hS6synAZSOoayV5w9exYTJ05Ungrj9u3baNSoEZo2bYrY2FhERUVhz549QpdZZThf6PVSU1OxZcsWREdHo1mzZvjggw9EE9ZfdeLECezfvx+ZmZmwsbFBw4YNERMTg8aNG2PSpEmiW97P4ZjXO3nyJMLCwuDk5IThw4eLZg7Vs2fPoKOjA0NDQxQVFWHPnj3YtWsXsrKy4Ovri27dusHb21s0ix44DaR0DGWvyM7Oxp49e3D79m106tQJPXr0gFQqRUFBgfKEdmLE+ULsRVWHGJf3c1WuKsV7Qm5uLiIiIlBUVARXV1fR9oAsXLgQxsbGcHFxQaNGjdCwYUMAz3ubpVKpaN5HAS6mexOGslLk5+fjwoULCAwMRFZWFrp27YpevXrB1NRU6NIEJfbLgbAXtXRc3v8ch2NUKYL54sWLER0djfj4eOTl5aFu3bpo3LgxJk6cKJoLk0ulUmWvemJiIp4+fQo3Nzc0bdoUzs7OsLS0RMOGDUXRQ6bAaSClYyh7iVQqRWFhoUp3+tWrV3H9+nU8ffoUPj4+ytUyYsD5QqrYi6qKy/tVcTimdD169MDWrVthbm6OgoICnD17FkFBQRg7dixat24tmuE6hVOnTmHZsmUYNGgQLl26hOTkZNjZ2WHJkiVClyYITgNRxVD2kitXrmDZsmV4+vQpbGxsYGtriydPnuDq1atISUmBpaUlTp06JXSZVY7zhV5gL+oLXN7/AodjVCUmJuL+/ftwc3PDtm3b4O/vL8prBb+ssLAQenp6+OWXX1CnTh2MGzcOwPNQ8u+//8LNzU0UAZXTQN6Moewl0dHRiI6ORs2aNREbGwsAqFu3LmrUqIGEhATUr18fvXr1ErhK4YlxvhB7UV/vo48+woABA9CnTx9l8PD398eMGTPQsmVL0YQRDse8EB4ejmnTpkFPTw8SiQTe3t6YOXOm8nQYYlNYWAhdXV1oaWlh7Nix+OKLL+Du7i7KBVKcBvJmDGX0Rpwv9Bx7UV+Py/tVcTgGymtcJicn4/jx4wgLC8PVq1dRo0YN1K9fH/PmzYOHh4fQZVaJhIQEfP755/Dz80OTJk2wYsUK7N+/X+iyBMNpIG/GUFZGYuhWfhnnC6liL6oqLu9/gcMxqoqKivD555+jR48e6N27t8pt0dHRCAwMxNChQ+Hg4CCK4yMrKws7d+7E9evXcefOHWRmZsLX1xeurq5o3rw5GjZsiJo1awpdZpXiNJDXYyijUnG+EL0Jl/e/wOEYVdnZ2diwYQOCgoKgra2NsWPHomvXrrC2tha6NI1w+/ZthIeH459//sG1a9cwbdo0TJw4UeiyqgSngbydOGZnk9rUuRyIGD5430QM3/ZfJpVKYW1tjejoaFy4cEH0y/ubNWuGuXPn4vbt22jRogW+/PJLUQ/H1KxZE9OnT8f06dOxY8cOXLp0Cenp6Rg8eLDoTgtSGmdnZzg7OwN4Pj0kNzcXgDjeR65du/bGaSCBgYGinQaiwJ4yeiPOF6I34fL+5zgc80JhYSGysrKU5zA8e/Ys1q1bh6ioKPj5+eGzzz6DlZWVwFWSEDgN5O0YyqgEzheit+Hy/uc4HKMqLS0NM2fORP369REfHw8dHR3lCtSdO3ciLi4O4eHhqF+/vtClEmkkhjIqgfOF6E24vP8FrspVdffuXcyYMQPu7u4YN24c5HI5Hj58CENDQ1hZWSElJQXt2rUTukzSQGL4ElcWDGWkgpcDoTfh8n5VHI4p6d69e8q5ZL6+vujfvz8aN26svJ1f6Ihej6GMXovzhehVXN5PZZWcnIyjR48iKysLHh4eXK1NVAYMZVQC5wtRWYl5ef/biO1vJCMjA/v374e5uTlsbW2Rm5uL2NhYXLt2Df/88w/MzMywefNm6OvrC10qkcbiKTFIhWK+EPB8wvIXX3wB4Pmwprm5OczNzQFAVB829HpiXt7/NmJ7/bdv30ZERARSU1ORkpICV1dXmJiY4PHjx8jOzkbDhg2hr6/P4UuiN2AoI6VX5wtlZmbC3d0dAEQ3gZvUp6enpzx/ndgCCQHe3t5o1qyZ8hJsSUlJkMvlypPGZmRkAOCcMqI34fAlKXG+EBFVNIYworJjKKPX4nwhIiKiqsNQRmWimC9kYmLC+UJERESVgKGMiIiISAOwu4OIiIhIAzCUEREREWkAhjIiIiIiDcBQRkRERKQBGMqIiIiINABDGRFpvDFjxuCnn34qsX3v3r1o1aqVxtRT2X799Vd88MEHVf68RFQ1GMqIiF6jqKhI6BKISEQYyoio2rhw4QKGDBkCT09PtGrVCiNGjMCjR4+Ut4eHh2PgwIFwd3dHly5dsGrVKhQXFytvd3R0xLZt2zBlyhR4enri999/L9Pz+vn54ffff8eXX36J5s2bw9fXFzt37lTenpiYCEdHR4SEhGDEiBFwd3dH3759cfHiReU+pfX6hYeHw9HRUXn7qlWrEB0dDUdHRzg6OmLv3r3laici0kwMZURULRQXF2Pq1Klo3bo19u/fj507d2L48OHK6y5evnwZc+bMwdixYxEaGooff/wRe/fuLRG8Vq1ahW7duuHAgQMYPHhwmZ9/w4YNcHNzQ3BwMEaNGoUffvgB9+/fV9nn559/xvjx4xEcHAxPT09MmTJFeaHut+nduzcmTJiApk2b4syZMzhz5gx69+5d5vqISPMxlBFRtZCdnY2srCx07twZdnZ2aNy4MQYOHAhra2sAz8PWpEmTMHDgQNja2qJdu3b49NNPsWPHDpXH6du3LwYPHgxbW1vlfcuiY8eOGD16NOzt7fHRRx+hTp06uHDhgso+o0ePRo8ePdC4cWP88MMPMDY2xp49e8r0+Pr6+jA0NIREIoGFhQUsLCygr69f5vqISPPpCF0AEVFFMDExwaBBgzBx4kS0a9cOPj4+6NWrFywtLQEA0dHR+Oeff1R6xqRSKQoKCpCXlwcDAwMAgJubW7meXzHMCABaWlowNzdHWlqayj7NmzdX/l9HRwdubm4letOISLwYyohI4xkZGSE7O7vE9mfPnsHY2Fj5c0BAAMaMGYPTp08jLCwMy5cvx4YNG+Dp6Ync3FxMnz4d3bt3L/E4NWrUUP7f0NCwXDXq6Ki+nWppaUGdSwtra2uX2J8LDYjEhcOXRKTxGjZsiKioqBLbb926hQYNGqhsc3FxweTJk7Fjxw44ODjg77//Vm5/8OAB7O3tS/zT1q6at8Jr164p/19cXIyoqCg0atQIAFCnTh3k5OQgNzdXuU90dLTK/XV1dSGTyaqkViKqeuwpIyKNN2rUKGzduhULFizAkCFDoKenh5MnTyIkJAS//fYbACAhIQG7du2Cn58fLC0t8eDBA8TFxSnP6zV16lRMmTIF1tbW6NGjB7S1tREdHY07d+5gxowZVfI6tm3bhgYNGqBRo0bYtGkTnj59qlxM0KxZMxgYGGDZsmUYO3Ysrl+/XmJ1pY2NDRITE3H79m1YWVmhZs2a0NPTq5LaiajyMZQRkcaztbXFX3/9heXLl2P8+PEoKipCo0aNsGLFCnTs2BEAYGBggPv37yMoKAiZmZmwtLTE6NGjMWLECABAhw4d8Pvvv2P16tX4888/oaOjg0aNGmHo0KFV9jpmzZqFNWvW4Pbt27C3t8dvv/0GU1NTAM/nxP33v//Fzz//jN27d8PHxwfTp0/Ht99+q7x/jx49cOTIEYwdOxbPnj1DQEAABg0aVGX1E1Hl0pKrM+mBiIjUlpiYiC5duiA4OBjOzs5Cl0NEGopzyoiIiIg0AEMZERERkQbg8CURERGRBmBPGREREZEGYCgjIiIi0gAMZUREREQagKGMiIiISAMwlBERERFpAIYyIiIiIg3AUEZERESkARjKiIiIiDQAQxkRERGRBvh/e2roY4HwA6EAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_metric('answer_relevancy')" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "79dc2a03-da9e-4f9d-8109-a94dfa79127e", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAHWCAYAAABACtmGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABRlUlEQVR4nO3deVhU5f//8ReLuKa5oJlLFgaigkiCSbiRlbmmpmapueSWVi591LQyt2izRds0jdQylxTL3RbNLDS3klwLF9wiRNMQFJm5f3/4Y75OoIcxcFCfj+vyqrnPfc55z+HcM/Oas4yHMcYIAAAAAHBJnu4uAAAAAAAKOoITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAABX4MyZMxo9erTuueceBQQEaOLEie4uycmiRYsUEBCg+Ph4d5eSp6ZMmaKAgAB3lwHgBuTt7gIA4Frw2Wefady4cQoODtaCBQvcXU6BkpGRoc8//1yxsbFKTEyUp6enKlSooNDQUPXo0UN+fn7uLjFfTJ06VbGxsXryySdVpUqVyz7PqKgoHTlyJMdpkZGRmjFjRn6VecWmTJmid999Vx4eHlqzZo0qVqzoND01NVURERE6d+6cHnvsMb344osur+PDDz9U9erV1axZs7wqGwDyDcEJAHJhyZIlqlSpkrZv366DBw/qtttuc3dJBcbTTz+tdevWqWXLlurYsaMyMzO1b98+rV27VnXr1r1ug9OGDRtUp04dDRo0KFf9AwMD1bNnz2zt5cuXz+vS8pSPj4+WLl2qPn36OLWvXr36Py976tSpeuCBB1wKTgMGDFDfvn3/87oBwFUEJwCwcOjQIW3btk3vvvuuXnzxRS1ZsiTXH5bzit1u1/nz51W4cOGrul4r27dv15o1azRkyBD179/faZrNZtPp06evWi3nzp1ToUKF5Ol5dc5CT0lJUfXq1XPdv0KFCmrbtm0+VpQ/GjdurGXLlmULTkuXLlWTJk20atWqq1JHWlqaihUrJm9vb3l78/EFwNXHNU4AYGHJkiUqVaqUGjdurAceeEBLlixxTDt//rzCw8P13HPPZZsvNTVVQUFBevXVVx1tGRkZmjx5su677z7Vrl1bjRs31muvvaaMjAyneQMCAjRu3Dh99dVXatmypYKCgvTDDz9IkmbMmKFHHnlE9evXV3BwsNq3b6+VK1dmW//Zs2c1YcIE1a9fX3Xr1lX//v2VlJSkgIAATZkyxalvUlKSnnvuOUVERKh27dpq2bKlvvjiC8ttc+jQIUlSaGhotmleXl4qXbp0tvWMGjVKkZGRql27tqKiojRmzBin53/o0CE9/fTTCg8PV506ddSpUyetXbvWaTkbN25UQECAli1bprfeeksNGzZUnTp1lJqaKkn69ddf1bt3b911112qU6eOunbtqi1btlg+H+lCIBo1apQiIiIUFBSkNm3aKDY2Ntu6Dx8+rLVr1yogIMDx+L/avXu3Ro4cqXvvvVdBQUG655579Nxzz+nkyZPZ+uZmW0oX9rno6GjdfffdCgkJ0cCBA3XixIlc19SqVSvt2rVLCQkJjrbk5GRt2LBBrVq1ynGe3OznAQEBSktLU2xsrGMbjhw5UtL/Xcf0xx9/aNiwYQoLC9Ojjz7qNO3fvvzySz388MOqU6eOwsLC9Nhjj2n9+vW5fp4AYIWvbADAwpIlS3TffffJx8dHrVq10ueff67t27crODhYhQoVUrNmzfT1119r7Nix8vHxccz3zTffKCMjQy1atJB04ajRgAEDtGXLFnXq1El+fn7au3evZs6cqQMHDuj99993Wu+GDRu0YsUKPfbYYypdurQqVaokSZo1a5aioqLUunVrnT9/XsuWLdMzzzyjqVOnqkmTJo75R44cqRUrVqht27aqU6eONm3alOMpTsePH1enTp3k4eGhxx57TGXKlNG6des0evRopaamqkePHpfcNrfeeqtjG4WGhl72SEBSUpIefvhh/fPPP+rUqZPuuOMOJSUladWqVTp79qx8fHx0/PhxPfLII0pPT1e3bt1UunRpxcbGasCAAY4P4hd7//33VahQIfXu3VsZGRkqVKiQ4uLi1KdPH9WuXVuDBg2Sh4eHFi1apMcff1xz5sxRcHDwJWs8e/asunXrpsTERD322GOqXLmyVq5cqZEjR+r06dN6/PHH5efnp9dee03R0dG65ZZbHKfflSlT5pLLlaTMzMwcA0uxYsVUpEgRSdJPP/2kQ4cOqX379vL19dXvv/+u+fPn648//tD8+fPl4eGR622ZZcKECSpZsqQGDRqkI0eOaObMmRo3bpzefvvty9abJSwsTLfccouWLl2qZ555RpK0fPlyFStWzGl/y5Lb/fy1117T888/r+DgYHXq1EmSVLVqVadlPfPMM7rttts0ZMgQGWMuWeO7776rKVOmqG7dunr66adVqFAh/frrr9qwYYMiIyNz9TwBwJIBAFxSfHy88ff3Nz/++KMxxhi73W4aNWpkJkyY4Ojzww8/GH9/f/Pdd985zdunTx9z7733Oh4vXrzY1KhRw2zatMmp3+eff278/f3Nli1bHG3+/v6mRo0a5vfff89WU3p6utPjjIwM06pVK9O9e3dH22+//Wb8/f3NxIkTnfqOHDnS+Pv7m8mTJzvaRo0aZe655x5z4sQJp75Dhgwxd911V7b1Xcxut5uuXbsaf39/ExERYYYOHWo+/fRTc+TIkWx9hw8fbmrUqGG2b9+e43KMMWbixInG39/faRulpqaaqKgo07RpU2Oz2YwxxmzYsMH4+/ube++916k+u91u7r//ftOrVy/HMrO2WVRUlOnZs+cln4sxxnzyySfG39/ffPnll462jIwM07lzZxMSEmL++ecfR3vTpk1N3759L7u8i/v6+/vn+G/q1KlOdf7b0qVLs22T3GzLhQsXGn9/f9OjRw+nbfHyyy+bwMBAc/r06cvWPHnyZOPv729SUlLMK6+8Yu677z7HtA4dOpiRI0caYy7sq2PHjnVMc2U/DwkJMSNGjLjkuocOHXrJaVkOHDhgatSoYQYOHOjYP/69LQAgL3CqHgBcxpIlS1SuXDnVr19fkuTh4aEWLVpo+fLlstlskqS7775bpUuX1vLlyx3znTp1Sj/99JPjaJMkrVy5Un5+frrjjjt04sQJx7+7775b0oVTwC4WFhaW4zU0WUcnstbzzz//6K677tLOnTsd7Vmn9WWd3pSla9euTo+NMVq9erWioqJkjHGqKzIyUv/884927Nhxye3j4eGhGTNmaPDgwSpZsqSWLl2qcePGqWnTpho8eLDjGie73a5vvvlGTZs2VVBQUI7LkaTvv/9ewcHBqlevnmNa8eLF1blzZx05ckR//PGH03wPPfSQ0/bYtWuXDhw4oNatW+vkyZOO55KWlqYGDRpo06ZNstvtl3w+69atk6+vr9MpaIUKFVK3bt2UlpamTZs2XXJeK3Xq1FFMTEy2fy1btnT0ufi5nDt3TidOnFCdOnUkyfF3yO22zJJ1NDFLvXr1ZLPZLnmXv5y0bt1aBw8edNwcJT4+Xq1bt86xr6v7+eU88sgjln2++eYb2e12DRw4MNv1bf/eFgDwX3CqHgBcgs1m07Jly1S/fn2n61eCg4P18ccfKy4uTpGRkfL29tb999+vpUuXKiMjQz4+Plq9erXOnz/vFJwOHjyohIQENWjQIMf1paSkOD2uXLlyjv3WrFmjDz74QLt27XK6ZuTiD4lHjx6Vp6dntmX8+26AJ06c0OnTpzVv3jzNmzcvx/VZXQ/j4+OjAQMGaMCAAfrrr7+0adMmzZo1SytWrJC3t7feeOMNnThxQqmpqbrzzjsvu6yjR486gsLF7rjjDsd0f39/R/u/n9+BAwckSSNGjLjkOv755x+VKlUqx2lHjhzRbbfdlu0DeNadAY8ePXrZ+i+ndOnSioiIuGyfv//+W++++66WL1+ebX/4559/JCnX2zJL1umUWUqWLClJLt24o2bNmrrjjju0dOlSlSxZUr6+vo4g9G+u7ueXc6kxcLGsW+Bfr3dvBFBwEJwA4BI2bNig5ORkLVu2TMuWLcs2fcmSJY7rJ1q2bKl58+Zp3bp1atasmVauXKk77rhDNWrUcPS32+3y9/fP8UYSknTLLbc4Pb746EOWzZs3a8CAAQoLC9OYMWPk6+urQoUKaeHChVq6dKnLzzHr6EubNm3Url27HPu48mOj5cuXV8uWLXX//ferVatWWrlypV555RWX68qtf28j8/+vgxk+fLgCAwNznKdYsWL5Vs9/NXjwYG3btk29e/dWYGCgihUrJrvdrieeeOKy1/hczqXuMujq8rKu7ytevLgefPDBSy7X1f38cgraXSQB3NgITgBwCUuWLFHZsmVz/GHPr7/+2nFDiCJFiigsLEy+vr5avny5QkNDtWHDhmy3565atap2796tBg0aXPEpRKtWrVLhwoU1Y8YMpxsALFy40KnfrbfeKrvdrsOHD6tatWqO9oMHDzr1K1OmjIoXLy673W55NMQVhQoVUkBAgA4cOKCTJ0+qbNmyKlGihH7//ffLznfrrbdq//792dr37dvnmH45VapUkSSVKFHiip5PpUqVtGfPHtntdqdgkNv1/xenTp1SXFycnnrqKafb3WcdRctSpkyZXG3LvNa6dWtNnjxZycnJev311y/ZLy/2c1dUrVpVdrtdCQkJlwzLAJAXuMYJAHJw9uxZrV69Wk2aNFHz5s2z/Xvsscd05swZfffdd5IufKvfvHlzrVmzRl999ZUyMzOdTtOTpAcffFBJSUmaP39+jutLS0uzrMvLy0seHh6O66sk6fDhw/r222+d+mUdCZszZ45T+6effppteQ888IBWrVqlvXv3Zluf1Wl6Bw4cyPH0tdOnT2vbtm0qVaqUypQpI09PTzVr1kxr1qxRfHx8tv5ZRz8aN26s7du3a9u2bY5paWlpmj9/vipVqmT5u0m1a9dW1apV9fHHH+vMmTMuP59GjRopOTnZ6Xq1zMxMzZ49W8WKFVNYWNhl5/8vvLy8cmyfOXOm0+Pcbsu8VrVqVY0aNUrDhg277J0JXdnPixUr9p9/66tZs2by9PTUe++9l+36tfzaFgBuTBxxAoAcfPfddzpz5oyioqJynB4SEqIyZcroq6++cgSkBx98ULNnz9bkyZPl7++f7ZqLtm3basWKFRozZow2btyo0NBQ2Ww27du3TytXrtT06dNzvNj/Yo0bN1ZMTIyeeOIJtWrVSikpKZozZ46qVq2qPXv2OPrVrl1bDzzwgGbOnKm///7bcTvyrKMXFx8JGDZsmDZu3KhOnTqpY8eOql69uk6dOqUdO3YoLi5OP//88yXr2b17t5599lk1bNhQ9erVU6lSpZSUlKTFixfrr7/+0qhRoxyBYOjQofrxxx/VrVs3x22qk5OTtXLlSs2ZM0clS5ZU3759HT+22q1bN5UqVUqLFy/W4cOHNWXKFMsft/X09NSECRPUp08ftWrVSu3bt1eFChWUlJSkjRs3qkSJEvrwww8vOX/nzp01b948jRw5Ujt27FClSpW0atUqbd26VaNGjVKJEiUuu/7LSUpK0pdffpmtvXjx4mrWrJlKlCihsLAwTZ8+XefPn1eFChX0448/5vj7ULnZlvnh8ccft+zjyn5eq1YtxcXFKSYmRuXLl1flypVzvMbtcm677Tb1799f77//vh599FHdf//98vHxUXx8vMqXL69hw4Zd0XMFgH8jOAFADr766isVLlxY99xzT47TPT091aRJEy1ZskQnT55U6dKlFRoaqooVK+rYsWPZjjZlzfPee+/pk08+0Zdffqmvv/5aRYsWVeXKldWtWzfdfvvtlnU1aNBAEydO1EcffaSXX35ZlStX1rPPPqsjR444BSdJevXVV1WuXDktW7ZMX3/9tSIiIvTWW2+pefPmTqf5lStXTgsWLNB7772nr7/+Wp9//rluvvlmVa9eXc8+++xl6wkLC9PTTz+tH374QTExMTp58qSKFy+uwMBAPfvss3rggQccfStUqKD58+frnXfe0ZIlS5SamqoKFSqoUaNGjmuVypUrp7lz5+r111/Xp59+qnPnzikgIEAffvhhjr8ZlJP69etr3rx5ev/99/Xpp58qLS1Nvr6+Cg4OVufOnS87b5EiRTR79my98cYbio2NVWpqqm6//XZFR0erffv2uVr/pezatUvDhw/P1l6pUiU1a9ZMkjRp0iSNHz9ec+bMkTFG99xzjz766CM1bNjQaZ7cbEt3cWU/HzlypF588UW9/fbbOnv2rNq1a+dycJIu/N5T5cqV9emnn+qtt95S0aJFFRAQoLZt2+blUwNwg/MwHMcGgBvGrl279NBDD+n1119XmzZt3F0OAADXDK5xAoDr1NmzZ7O1zZw5U56envl6rQ4AANcjTtUDgOvU9OnT9dtvv+nuu++Wl5eX1q1bp3Xr1qlz586qWLGiu8sDAOCawql6AHCd+vHHH/Xuu+8qISFBaWlpqlixotq2bav+/fvL25vvzQAAcAXBCQAAAAAscI0TAAAAAFggOAEAAACAhRvuJHe73a7MzEx5eno6/QAkAAAAgBuLMUZ2u13e3t6WP7J+wwWnzMxMxcfHu7sMAAAAAAVEUFCQ04/D5+SGC05ZSTIoKEheXl5urgYAAACAu9hsNsXHx1sebZJuwOCUdXqel5cXwQkAAABAri7h4eYQAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFtwanDZt2qT+/fsrMjJSAQEB+uabbyzn2bhxo9q1a6fatWvrvvvu06JFi65CpQAAAABuZG4NTmlpaQoICNCYMWNy1f/QoUPq16+f6tevry+//FKPP/64nn/+ef3www/5XCkAAACAG5m3O1feuHFjNW7cONf9586dq8qVK2vkyJGSJD8/P23ZskWffPKJGjZsmF9lAgAAALjBuTU4ueqXX35RgwYNnNoiIyP18ssvu7wsm82WV2UBAAAAuAa5kgmuqeB0/PhxlStXzqmtXLlySk1N1dmzZ1WkSJFcLys+Pj6vywMAAABwnbqmglNeCgoKkpeXl7vLAAAAAOAmNpst1wdUrqngVK5cOR0/ftyp7fjx4ypRooRLR5skycvLi+AEAAAAIFeuqd9xCgkJ0YYNG5zafvrpJ4WEhLinIAAAAAA3BLcGpzNnzmjXrl3atWuXJOnw4cPatWuXjh49KkmaNGmShg8f7uj/yCOP6NChQ3rttdeUkJCgzz77TCtWrFCPHj3cUT4AAACAG4RbT9X77bff1L17d8fj6OhoSVK7du30yiuvKDk5WceOHXNMr1KliqZOnaro6GjNmjVLt9xyiyZMmMCtyAEAAADkKw9jjHF3EVeTzWbTL7/8opCQEK5xAgAAAG5grmSDa+oaJwAAAABwB4ITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAB5wGa3u7sE3CDY1wD38HZ3AQAAXA+8PD31/LcLtP9ksrtLwXXs9tK+mnBvR3eXAdyQCE4AAOSR/SeTtef4MXeXAQDIB5yqBwAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAAAAWCE4AAAAAYIHgBAAAgDxhN3Z3l4AbhDv2Ne+rvkYAAABclzw9PLVo9zQlpx11dym4jvkWu1Xta/S96uslOOUBu90uT08O3iH/sa8BAAq65LSj+jM10d1lAHmO4JQHPD099fobX+jQ4ePuLgXXsSqVy+l/zz7s7jIAAABuSASnPHLo8HElJBxzdxkAAAAA8gHn/AAAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFhwe3D67LPPFBUVpaCgIHXs2FHbt2+/bP9PPvlEDzzwgIKDg9W4cWO9/PLLOnfu3FWqFgAAAMCNyK3Bafny5YqOjtbAgQMVGxurGjVqqHfv3kpJScmx/5IlSzRp0iQNGjRIy5cv18SJE7V8+XK9+eabV7lyAAAAADcStwanmJgYderUSR06dFD16tU1duxYFSlSRAsXLsyx/7Zt2xQaGqrWrVurcuXKioyMVKtWrSyPUgHIXza73d0l4AbBvgYAcBdvd604IyNDO3bsUL9+/Rxtnp6eioiI0LZt23Kcp27duvrqq6+0fft2BQcH69ChQ/r+++/Vtm1bl9dvs9muuPZ/8/LyyrNlAVbyct/NK15eXnrx01gdSDru7lJwHatWoZzGdW1XIMeAxHsBri7GAZA348CVZbgtOJ08eVI2m01ly5Z1ai9btqz27duX4zytW7fWyZMn9eijj8oYo8zMTD3yyCPq37+/y+uPj4+/orr/rWjRoqpZs2aeLAvIjT179ig9Pd3dZThkjYEDSce158if7i4HN4CCNgYk3gtw9TEOgKs/DtwWnK7Exo0bNXXqVI0ZM0bBwcFKTEzUxIkT9d5772ngwIEuLSsoKIhvRXBNCggIcHcJgFsxBgDGASDlzTiw2Wy5PqDituBUunRpeXl5ZbsRREpKisqVK5fjPO+8847atGmjjh07SrqwsdLS0vTiiy9qwIAB8vTM/SVbXl5eBCdck9hvcaNjDACMA0C6+uPAbTeH8PHxUa1atRQXF+dos9vtiouLU926dXOc5+zZs9nCUdYGM8bkX7EAAAAAbmhuPVWvZ8+eGjFihGrXrq3g4GDNnDlT6enpat++vSRp+PDhqlChgoYNGyZJatq0qWJiYlSzZk3HqXrvvPOOmjZtyjcvAAAAAPKNW4NTixYtdOLECU2ePFnJyckKDAzU9OnTHafqHTt2zOkI04ABA+Th4aG3335bSUlJKlOmjJo2baohQ4a46ykAAAAAuAG4/eYQXbt2VdeuXXOcNnv2bKfH3t7eGjRokAYNGnQ1SgMAAAAASW7+AVwAAAAAuBYQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAwhUFp82bN+vZZ59V586dlZSUJElavHixNm/enKfFAQAAAEBB4HJwWrVqlXr37q0iRYpo586dysjIkCSlpqZq6tSpeV4gAAAAALiby8Hpgw8+0NixYzVhwgR5e3s72kNDQ7Vz5848LQ4AAAAACgKXg9P+/ftVr169bO033XSTTp8+nSdFAQAAAEBB4nJwKleunBITE7O1b9myRVWqVMmTogAAAACgIHE5OHXq1EkTJ07Ur7/+Kg8PDyUlJemrr77Sq6++qi5duuRHjQAAAADgVt7WXZz17dtXdrtdPXr0UHp6urp27SofHx/16tVL3bp1y48aAQAAAMCtXApONptNW7du1WOPPabevXsrMTFRaWlp8vPzU/HixfOrRgAAAABwK5dO1fPy8lKvXr106tQp+fj4qHr16goODiY0AQAAALiuuXyN05133qnDhw/nRy0AAAAAUCC5HJwGDx6sV199VWvWrNFff/2l1NRUp38AAAAAcL25optDSNKAAQPk4eHhaDfGyMPDQ7t27cq76gAAAACgAHA5OM2aNSs/6gAAAACAAsvl4BQeHp4fdQAAAABAgeVycJKk06dP64svvlBCQoKkCzeM6NChg2666aY8LQ4AAAAACgKXbw4RHx+v++67T5988olOnTqlU6dOKSYmRs2aNdOOHTvyo0YAAAAAcCuXjzhFR0crKipK48ePl7f3hdkzMzP1/PPP6+WXX9Znn32W50UCAAAAgDu5fMTpt99+0xNPPOEITZLk7e2tJ554Qr/99lueFgcAAAAABYHLwalEiRI6duxYtvZjx46pePHieVIUAAAAABQkLgenFi1aaPTo0Vq+fLmOHTumY8eOadmyZXr++efVsmXL/KgRAAAAANzK5Wuchg8f7vivzWa7sBBvb3Xp0kXPPvts3lYHAAAAAAWAy8HJx8dHzz//vIYNG6bExERJUtWqVVW0aNE8Lw4AAAAACgKXg9M///wjm82mm2++WQEBAY72v//+W97e3ipRokSeFggAAAAA7ubyNU5DhgzRsmXLsrWvWLFCQ4YMyZOiAAAAAKAgcTk4bd++XXfffXe29vDwcG3fvj1PigIAAACAgsTl4JSRkaHMzMxs7ZmZmTp79myeFAUAAAAABYnLwSkoKEjz58/P1j537lzVqlUrT4oCAAAAgILE5ZtDDB48WD179tTu3bvVoEEDSVJcXJzi4+P18ccfu1zAZ599phkzZig5OVk1atTQCy+8oODg4Ev2P336tN566y19/fXX+vvvv1WpUiWNGjVKjRs3dnndAAAAAJAbLgenu+66S/PmzdP06dO1YsUKFS5cWAEBAZo4caKqVavm0rKWL1+u6OhojR07VnXq1NHMmTPVu3dvrVy5UmXLls3WPyMjQz179lTZsmX1zjvvqEKFCjp69KhKlizp6tMAAAAAgFxzOThJUmBgoCZNmvSfVx4TE6NOnTqpQ4cOkqSxY8dq7dq1Wrhwofr27Zut/8KFC3Xq1CnNnTtXhQoVkiRVrlz5P9cBAAAAAJeT6+CUmZkpu90uHx8fR9vx48c1d+5cpaWlKSoqSvXq1cv1ijMyMrRjxw7169fP0ebp6amIiAht27Ytx3m+++47hYSEaNy4cfr2229VpkwZtWrVSn369JGXl1eu1y1JNpvNpf6X4+q6gf8iL/fdvMIYwNVUEMeAxDjA1cU4APJmHLiyjFwHpxdeeEGFChXSuHHjJEmpqal6+OGHde7cOfn6+mrmzJl6//33c32t0cmTJ2Wz2bKdkle2bFnt27cvx3kOHTqkDRs2qHXr1po2bZoSExM1duxYZWZmatCgQbl9KpKk+Ph4l/pfStGiRVWzZs08WRaQG3v27FF6erq7y3BgDOBqK2hjQGIc4OpjHABXfxzkOjht3bpVL7zwguPxl19+KZvNptWrV+umm27S66+/runTp+frTRqMMSpbtqzGjx8vLy8v1a5dW0lJSZoxY4bLwSkoKIhvRXBNCggIcHcJgFsxBgDGASDlzTiw2Wy5PqCS6+CUlJSk2267zfE4Li5ODzzwgG666SZJUrt27bRo0aJcF1m6dGl5eXkpJSXFqT0lJUXlypXLcR5fX195e3s7BZ477rhDycnJysjIcDqN0IqXlxfBCdck9lvc6BgDAOMAkK7+OMj17zgVLlxY586dczz+5ZdfnG4bXrhwYaWlpeV6xT4+PqpVq5bi4uIcbXa7XXFxcapbt26O84SGhioxMVF2u93RduDAAfn6+roUmgAAAADAFbkOTjVq1NCXX34pSdq8ebNSUlJ09913O6YnJiaqfPnyLq28Z8+emj9/vmJjY5WQkKCXXnpJ6enpat++vSRp+PDhTnfv69Kli/7++29NnDhR+/fv19q1azV16lQ99thjLq0XAAAAAFyR61P1Bg4cqD59+mjFihVKTk5Wu3btnILS119/rdDQUJdW3qJFC504cUKTJ09WcnKyAgMDNX36dMepeseOHZOn5/9lu4oVK2rGjBmKjo5WmzZtVKFCBXXv3l19+vRxab0AAAAA4IpcB6fw8HAtWrRI69evl6+vr5o3b+40PTAw0OnUvdzq2rWrunbtmuO02bNnZ2urW7eu5s+f7/J6AAAAAOBKufQDuH5+fvLz88txWufOnfOkIAAAAAAoaHJ9jRMAAAAA3KgITgAAAABggeAEAAAAABYITgAAAABg4YqC0+nTp7VgwQJNmjRJf//9tyRpx44dSkpKysvaAAAAAKBAcOmuepK0e/du9ezZUzfddJOOHDmiTp066eabb9bq1at17Ngxvfbaa/lRJwAAAAC4jctHnF555RW1a9dOq1evlo+Pj6O9cePG2rx5c54WBwAAAAAFgcvBKT4+Xo888ki29goVKig5OTlPigIAAACAgsTl4OTj46PU1NRs7QcOHFCZMmXypCgAAAAAKEhcDk5RUVF67733dP78eUfb0aNH9cYbb+j+++/P0+IAAAAAoCBwOTiNHDlSaWlpioiI0Llz59StWzfdf//9Kl68uIYMGZIfNQIAAACAW7l8V72bbrpJMTEx2rx5s/bs2aO0tDTVqlVLERER+VEfAAAAALidy8EpS7169VSvXr28rAUAAAAACiSXg9OsWbNybPfw8FDhwoVVtWpVhYWFycvL6z8XBwAAAAAFgcvB6ZNPPtHJkyeVnp6uUqVKSZJOnTqlokWLqlixYkpJSVGVKlU0a9YsVaxYMc8LBgAAAICrzeWbQwwdOlS1a9fW6tWrtXHjRm3cuFGrVq1ScHCwRo8erbVr16pcuXKKjo7Oj3oBAAAA4KpzOTi9/fbbGjVqlKpWrepou+222zRixAhNmjRJt9xyi/73v/9p69ateVooAAAAALiLy8EpOTlZmZmZ2dozMzN1/PhxSVL58uV15syZ/14dAAAAABQALgen+vXra8yYMdq5c6ejbefOnXrppZd09913S5L27t2rypUr512VAAAAAOBGLt8cYuLEiRo+fLjat28vb+8Ls9tsNjVo0EATJ06UJBUrVkwjRozI20oBAAAAwE1cDk6+vr6KiYlRQkKCDhw4IEm6/fbbdccddzj6ZB15AgAAAIDrwRX/AK6fn5/8/PzyshYAAAAAKJCuKDj9+eef+vbbb3Xs2DGdP3/eadpzzz2XJ4UBAAAAQEHhcnCKi4vTgAEDVKVKFe3bt0933nmnjhw5ImOMatasmR81AgAAAIBbuXxXvUmTJqlXr15asmSJfHx8NGXKFK1du1ZhYWFq3rx5ftQIAAAAAG7lcnBKSEjQQw89JEny9vbW2bNnVbx4cT3zzDOaPn16XtcHAAAAAG7ncnAqVqyY47omX19fJSYmOqadPHky7yoDAAAAgALC5Wuc6tSpoy1btsjPz0+NGzfWq6++qr179+rrr79WnTp18qNGAAAAAHArl4PTc889pzNnzkiSnnrqKZ05c0bLly9XtWrVNHLkyDwvEAAAAADczaXgZLPZ9OeffyogIEDShdP2xo0bly+FAQAAAEBB4dI1Tl5eXurVq5dOnTqVX/UAAAAAQIHj8s0h7rzzTh0+fDg/agEAAACAAsnl4DR48GC9+uqrWrNmjf766y+lpqY6/QMAAACA643LN4fo27evJGnAgAHy8PBwtBtj5OHhoV27duVddQAAAABQALgcnGbNmpUfdQAAAABAgeVycAoPD8+POgAAAACgwHL5GidJ2rx5s5599lk98sgjSkpKkiQtXrxYmzdvztPiAAAAAKAgcDk4rVq1Sr1791aRIkW0Y8cOZWRkSJJSU1M1derUPC8QAAAAANzN5eD0wQcfaOzYsZowYYK8vf/vTL/Q0FDt3LkzT4sDAAAAgILA5eC0f/9+1atXL1v7TTfdpNOnT+dJUQAAAABQkLgcnMqVK6fExMRs7Vu2bFGVKlXypCgAAAAAKEhcDk6dOnXSxIkT9euvv8rDw0NJSUn66quv9Oqrr6pLly75USMAAAAAuNUV/QCu3W5Xjx49lJ6erq5du8rHx0e9evVSt27d8qNGAAAAAHArl4OTh4eHBgwYoN69eysxMVFpaWny8/NT8eLF86M+AAAAAHA7l0/V+/LLL5Weni4fHx9Vr15dwcHBhCYAAAAA1zWXg1N0dLQiIiI0bNgwff/997LZbPlRFwAAAAAUGC6fqrd+/Xr98MMPWrp0qQYPHqwiRYqoefPmat26tUJDQ/OjRgAAAABwK5eDk7e3t5o2baqmTZsqPT1dX3/9tZYuXaru3bvrlltu0TfffJMfdQIAAACA27gcnC5WtGhRRUZG6vTp0zp69KgSEhLyqi4AAAAAKDCuKDhlHWlasmSJ4uLiVLFiRbVs2VLvvPNOXtcHAAAAAG7ncnAaMmSI1q5dqyJFiujBBx/Uk08+qbp16+ZHbQAAAABQILgcnDw9PfX2228rMjJSXl5eTtP27t0rf3//PCsOAAAAAAoCl4PTpEmTnB6npqZq2bJlWrBggXbs2KFdu3blWXEAAAAAUBBc8c0hNm3apC+++EKrV69W+fLldd999+nFF1/My9oAAAAAoEBwKTglJycrNjZWX3zxhVJTU/Xggw8qIyND7733nqpXr55fNQIAAACAW+U6OPXv31+bNm1SkyZNNGrUKDVs2FBeXl6aO3duftYHAAAAAG6X6+C0bt06devWTV26dFG1atXysSQAAAAAKFg8c9txzpw5OnPmjNq3b6+OHTvq008/1YkTJ/KzNgAAAAAoEHIdnEJCQjRhwgStX79enTt31rJly9SoUSPZ7Xb9+OOPSk1NveIiPvvsM0VFRSkoKEgdO3bU9u3bczXfsmXLFBAQoCeffPKK1w0AAAAAVnIdnLIUK1ZMDz/8sD7//HN99dVX6tmzpz766CNFRESof//+LhewfPlyRUdHa+DAgYqNjVWNGjXUu3dvpaSkXHa+w4cP69VXX1W9evVcXicAAAAAuMLl4HSxO+64Q8OHD9f333+vN99884qWERMTo06dOqlDhw6qXr26xo4dqyJFimjhwoWXnMdms+nZZ5/VU089pSpVqlxp+QAAAACQK1f8O04X8/LyUrNmzdSsWTOX5svIyNCOHTvUr18/R5unp6ciIiK0bdu2S8733nvvqWzZsurYsaO2bNlyRTXbbLYrmi8nXl5eebYswEpe7rt5hTGAq6kgjgGJcYCri3EA5M04cGUZeRKcrtTJkydls9lUtmxZp/ayZctq3759Oc6zefNmffHFF1q8ePF/Wnd8fPx/mj9L0aJFVbNmzTxZFpAbe/bsUXp6urvLcGAM4GoraGNAYhzg6mMcAFd/HLg1OLkqNTVVw4cP1/jx41WmTJn/tKygoCC+FcE1KSAgwN0lAG7FGAAYB4CUN+PAZrPl+oCKW4NT6dKl5eXlle1GECkpKSpXrly2/ocOHdKRI0c0YMAAR5vdbpck1axZUytXrlTVqlVztW4vLy+CE65J7Le40TEGAMYBIF39ceDW4OTj46NatWopLi7OcX2U3W5XXFycunbtmq3/HXfcoSVLlji1vf322zpz5oxGjx6tW2655arUDQAAAODG4vZT9Xr27KkRI0aodu3aCg4O1syZM5Wenq727dtLkoYPH64KFSpo2LBhKly4sPz9/Z3mL1mypCRlawcAAACAvOL24NSiRQudOHFCkydPVnJysgIDAzV9+nTHqXrHjh2Tp+d/ums6AAAAAPwnbg9OktS1a9ccT82TpNmzZ1923ldeeSU/SgIAAAAABw7lAAAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAICFAhGcPvvsM0VFRSkoKEgdO3bU9u3bL9l3/vz5evTRRxUWFqawsDD16NHjsv0BAAAA4L9ye3Bavny5oqOjNXDgQMXGxqpGjRrq3bu3UlJScuy/ceNGtWzZUrNmzdLcuXNVsWJF9erVS0lJSVe5cgAAAAA3CrcHp5iYGHXq1EkdOnRQ9erVNXbsWBUpUkQLFy7Msf+kSZP02GOPKTAwUH5+fpowYYLsdrvi4uKucuUAAAAAbhTe7lx5RkaGduzYoX79+jnaPD09FRERoW3btuVqGenp6crMzFSpUqVcWrfNZnOp/+V4eXnl2bIAK3m57+YVxgCupoI4BiTGAa4uxgGQN+PAlWW4NTidPHlSNptNZcuWdWovW7as9u3bl6tlvPHGGypfvrwiIiJcWnd8fLxL/S+laNGiqlmzZp4sC8iNPXv2KD093d1lODAGcLUVtDEgMQ5w9TEOgKs/DtwanP6radOmafny5Zo1a5YKFy7s0rxBQUF8K4JrUkBAgLtLANyKMQAwDgApb8aBzWbL9QEVtwan0qVLy8vLK9uNIFJSUlSuXLnLzjtjxgxNmzZNMTExqlGjhsvr9vLyIjjhmsR+ixsdYwBgHADS1R8Hbr05hI+Pj2rVquV0Y4esGz3UrVv3kvN99NFHev/99zV9+nQFBQVdjVIBAAAA3MDcfqpez549NWLECNWuXVvBwcGaOXOm0tPT1b59e0nS8OHDVaFCBQ0bNkzShdPzJk+erEmTJqlSpUpKTk6WJBUrVkzFixd32/MAAAAAcP1ye3Bq0aKFTpw4ocmTJys5OVmBgYGaPn2641S9Y8eOydPz/w6MzZ07V+fPn9fTTz/ttJxBgwbpqaeeuqq1AwAAALgxuD04SVLXrl3VtWvXHKfNnj3b6fF33313NUoCAAAAAAe3/wAuAAAAABR0BCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBCcAAAAAsEBwAgAAAAALBSI4ffbZZ4qKilJQUJA6duyo7du3X7b/ihUr1Lx5cwUFBal169b6/vvvr1KlAAAAAG5Ebg9Oy5cvV3R0tAYOHKjY2FjVqFFDvXv3VkpKSo79t27dqmHDhunhhx/W4sWLde+992rgwIHau3fvVa4cAAAAwI3C7cEpJiZGnTp1UocOHVS9enWNHTtWRYoU0cKFC3PsP2vWLDVs2FBPPPGE/Pz8NHjwYNWsWVOffvrpVa4cAAAAwI3C250rz8jI0I4dO9SvXz9Hm6enpyIiIrRt27Yc5/nll1/Uo0cPp7bIyEh98803uVqnMcaxbi8vrysr/F+8vLxU7TZfFSrk9hyK61ilW8vKZrPJZrO5u5RsvLy8VL2ir3y8GAPIP1XLF9wxIF0YB3eWLi8fj7x5bwFyctvNBX8clC9aWZ7GrR8xcZ0rV/SWPBsHWcvIygiX49a9+uTJk7LZbCpbtqxTe9myZbVv374c5zl+/LjKlSuXrf/x48dztU673S5J2rlz5xVUfGmNG1WTVC1Plwn82y+//OLuEi6pZUAVKaCKu8vAda4gjwFJan3z7dLNt7u7DFznCvo4qKq6quruInB9S8v7cZCVES7nhvs6wNvbW0FBQfL09JSHh4e7ywEAAADgJsYY2e12eXtbxyK3BqfSpUvLy8sr240gUlJSsh1VylKuXLlsR5cu1//fPD095ePjc2UFAwAAALghufWCBB8fH9WqVUtxcXGONrvdrri4ONWtWzfHeUJCQrRhwwantp9++kkhISH5WSoAAACAG5jbr+Tu2bOn5s+fr9jYWCUkJOill15Senq62rdvL0kaPny4Jk2a5OjfvXt3/fDDD/r444+VkJCgKVOm6LffflPXrl3d9RQAAAAAXOfcfo1TixYtdOLECU2ePFnJyckKDAzU9OnTHafeHTt2TJ6e/5fvQkND9cYbb+jtt9/Wm2++qWrVqum9996Tv7+/u54CAAAAgOuch8nNvfcAAAAA4Abm9lP1AAAAAKCgIzgBAAAAgAWCEwAAAABYIDjhhhIVFaVPPvkkz/sC16vDhw8rICBAu3btkiRt3LhRAQEBOn36tJsrgyuMMXrhhRcUHh7u9Pe8lH//3S+lW7dumjhx4mX7JCcnq2fPngoJCVG9evVyVS/7GZDdyJEj9eSTT7q7jBsawekGkl9B4FoKGF988YU6d+6c531x/WLc4Hqwbt06xcbG6sMPP9T69et15513XrZ/xYoVnfr9lyDzySefKDk5WYsXL9aqVauuqH4AKAjcfjtyIDcyMjLk4+Pzn5dTpkyZfOkLXG02m00eHh5OP9cAXMqhQ4fk6+ur0NDQXPX38vKSr69vnq27Vq1aqlatWp4sDyjIruS1Oa8+4yD/8Y5bgNjtdn300Ue67777VLt2bTVp0kQffPCBJGnPnj3q3r27goODVb9+fb3wwgs6c+aMY96sw7czZsxQZGSk6tevr7Fjx+r8+fOSLpxOceTIEUVHRysgIEABAQGOeTdv3qxHH31UwcHBaty4sSZMmKC0tDRJ0uLFi1W3bl0dOHDA0f+ll15S8+bNlZ6eftnlXsqiRYtUr149ffPNN7r//vsVFBSk3r1769ixY44+U6ZMUdu2bbVgwQJFRUUpODhYknT69GmNHj1ad999t0JDQ9W9e3ft3r3bafnfffedOnTooKCgINWvX18DBw50TLv4W35jjKZMmaImTZqodu3aioyM1IQJE3LsK0lHjx7VgAEDVLduXYWGhuqZZ57R8ePHs9W8ePFiRUVF6a677tKQIUOUmppquU1w5W60cfPtt9+qRYsWCgoK0tGjR5WRkaFXX31VDRs2VEhIiDp27KiNGzc6zbtlyxZ169ZNderUUVhYmHr37q1Tp05JunAkokuXLqpXr57q16+vfv36KTEx8cr+GCiQRo4cqfHjx+vo0aMKCAhQVFSU5d/94lP1Dh8+rO7du0uSwsLCFBAQoJEjRzr6GmP02muvKTw8XPfcc4+mTJnimBYVFaVVq1Zp8eLFjvlyOg3w9OnTCggIyLbvZsna/3/44Qc9+OCDqlu3rnr37q2//vrLqd+CBQv04IMPKigoSM2bN9dnn33mmJaRkaFx48YpMjJSQUFBatq0qaZOnep4Dpd7P8DVdbn9M2v/Wb16teN1rU2bNtq2bZtj/iNHjqh///4KCwtTSEiIWrZsqe+//16S1L59e82YMcPR98knn1StWrUc7w1//vmnAgICdPDgQUmyfI291Gvz5WS993zwwQeKjIxU8+bNJV347dJnnnlG9erVU3h4uAYMGKDDhw9fcjl2u11Tp051fE5q06aNVq5c6ZjWqFEjzZkzx2menTt3qkaNGjpy5IgkKSYmRq1bt1ZISIgaN26sl156yel9Mrdj74svvlDLli0d42fcuHGSpOeee079+vVz6nv+/Hk1aNBACxYsuOx2KpAMCozXXnvNhIWFmUWLFpmDBw+aTZs2mfnz55szZ86Ye+65xwwaNMjs2bPH/PTTTyYqKsqMGDHCMe+IESNMaGioefHFF80ff/xhvvvuO1OnTh0zb948Y4wxJ0+eNI0aNTLvvvuu+euvv8xff/1ljDHm4MGDJiQkxMTExJj9+/ebLVu2mIceesiMHDnSseynn37adOjQwZw/f96sWbPG1KpVy8THx192uZezcOFCU6tWLdO+fXuzdetWEx8fbx5++GHTuXNnR5/JkyebkJAQ07t3b7Njxw6za9cuY4wxPXr0MP369TPbt283+/fvN6+88ooJDw83J0+eNMYYs2bNGhMYGGjeeecd88cff5hdu3aZDz/80LHcpk2bmpiYGGOMMStWrDChoaFm7dq15siRI+bXX391bK9/97XZbKZt27amS5cuJj4+3vzyyy+mXbt2pmvXrtlqzvo7bdq0ydxzzz3mzTfftNwmuHI32rjp3Lmz2bJli0lISDBpaWlm9OjRpnPnzmbTpk3m4MGDZvr06aZ27dpm//79xhhjdu7caWrXrm3GjBljdu3aZfbu3Wtmz55tUlJSjDHGrFy50qxatcocOHDA7Ny50/Tr18+0atXK2Gw2Y4wxhw4dMv7+/mbnzp3GGGM2bNhg/P39zalTp/7jXw5Xy+nTp827775rGjVqZP766y+TkpLi0t89MzPTrFq1yvj7+5t9+/aZv/76y5w+fdoYY0zXrl1NaGiomTJlitm/f7+JjY01AQEBZv369cYYY1JSUkzv3r3NM88845jv3/uUMcacOnXK+Pv7mw0bNhhjsu9nWft/jx49zPbt281vv/1mHnzwQTN06FDHMr788ktzzz33mFWrVpnExESzatUqEx4ebhYtWmSMMWb69OmmcePGZtOmTebw4cNm06ZNZsmSJcYY6/cDXF2X2z+z9p/mzZubNWvWmH379pmnnnrKNG3a1Jw/f94YY0zfvn1Nz549ze7du01iYqL57rvvzM8//2yMMSY6Otr07dvXGGOM3W434eHhpn79+ub77783xlzYjxo2bOioxeo19lKvzZczYsQIExISYv73v/+ZvXv3mr1795qMjAzz4IMPmueee87s3r3b/PHHH2bo0KHmgQceMOfOnXPMN2DAAMdy3n//fdO8eXOzbt06k5iYaBYuXGhq165tNm7caIwx5pVXXjFdunRxWve/22JiYkxcXJw5dOiQ+emnn8wDDzxgxowZ45iem7H32WefmaCgIPPJJ5+Yffv2mV9//dXx+WnLli0mMDDQJCUlOfqvXr3ahISEmNTU1Mtup4KI4FRA/PPPP6Z27dpm/vz52abNmzfPhIWFmTNnzjja1q5da2rUqGGSk5ONMRcGU9OmTU1mZqajz9NPP20GDx7seHxxEMgyatQo88ILLzi1bdq0ydSoUcOcPXvWGGPM33//bRo1amTGjBljIiIizAcffODUP6flXs7ChQuNv7+/+eWXXxxtf/zxh/H39ze//vqrMeZCCKlVq5bjw11WXaGhoY4XkCzNmjUzc+fONcYY07lzZzNs2LBLrvviWj/++GNz//33m4yMDMu+69evN4GBgebo0aOO6b///nu2muvUqWP++ecfR59XX33VdOzY0WqT4ArdiOMm60sEY4w5cuSICQwMNH/++adT38cff9xMmjTJGGPM0KFDzSOPPJLr9aSkpBh/f3+zZ88eYwzB6XoRExNjmjZtesnpV/p379q1a7YPZh06dDCvv/664/GAAQOcvrC40uDk7+9vDh486Jjn008/NREREY7HzZo1cwShLO+9957jS7nx48eb7t27G7vdnu35W70fwL0u3j+z9p+LX/ez3o//+OMPY4wxrVq1MlOmTMlxWd9++6256667TGZmptm1a5e55557zIQJExz77OjRox2hIDevsTm9NlsZMWKEiYiIcPo8s3jxYvPAAw847Z/nzp0zwcHB5ocffnDMlxWczp07Z+rUqWO2bt3qtOxRo0Y56t+5c6cJCAgwR44cMcZc+BK4YcOGZs6cOZesbcWKFSY8PNzxODdjLzIy8rJfErdo0cJMmzbN8bhfv35OXzReS7jGqYDYt2+fMjIydPfdd2eblpCQoICAABUrVszRFhoaKrvdrv3796tcuXKSpOrVq8vLy8vRx9fXV3v37r3senfv3q09e/ZoyZIljjZjjOx2uw4fPiw/Pz+VKlVKEydOVO/evVW3bl317dv3vz5deXt7KygoyPHYz89PJUuWVEJCguO0vFtvvdXpOqM9e/YoLS1N9evXd1rW2bNnHYfwd+3apY4dO+aqhubNm2vmzJlq1qyZGjZsqMaNG6tp06by9s4+LBISEnTLLbeoYsWKjrbq1aurZMmS2rdvn6PmSpUqqUSJEo4+5cuXV0pKSq7qgetutHFTqFAhp9P69u7dK5vN5jjNI0tGRoZuvvlmSRfGxL+nX+zAgQOaPHmyfv31V508eVLGGEkXThnx9/f/zzWjYMrLv/u/TzX19fXNl9e9okWLqmrVqo7HF7++pqWlKTExUaNHj9YLL7zg6JOZmambbrpJktSuXTv16tVLzZs3V8OGDdWkSRNFRkZKcu39APnvcvunn5+fJOf9Lut6vBMnTsjPz0/du3fXSy+9pPXr1ysiIkL333+/atSoIUmqV6+ezpw5o507d2rbtm0KCwtT/fr1NW3aNEnSpk2b1Lt3b0m5e42Vsr8254a/v7/TdU27d+9WYmJitusQz507l+Pp0wcPHlR6erp69erl1H7+/HkFBgZKkgIDA+Xn56elS5eqb9+++vnnn3XixAmn5/PTTz9p6tSp2rdvn1JTU2Wz2XTu3Dmlp6eraNGiki4/9lJSUvTXX3+pQYMGl3yuHTt21Lx589SnTx8dP35cP/zwg2bOnJnbTVWg8IpQQBQuXPg/L+PfL/AeHh6OF5tLSUtL0yOPPKJu3bplm3ZxSNi0aZO8vLyUnJystLQ0p3CQX7IGbJYzZ87I19dXs2fPztY3642xSJEiuV5+xYoVtXLlSv3000/66aefNHbsWM2YMUOzZ89WoUKFrqjmnN5krf4GuHI32rgpUqSIPDw8nOrw8vLSwoULncKfJEdgtBoT/fv3V6VKlTRhwgSVL19edrtdrVq1clznhetTXv7dXR1DWRfNX9wnMzPzP60n6/rC8ePHq06dOjmur1atWvr222+1bt06/fTTTxo8eLAiIiI0efLkfHk/wJXLzf558d8l63XRbrdLuvBBPTIyUmvXrtWPP/6oadOmacSIEerWrZtKliypGjVq6Oeff9Yvv/yiiIgI1atXT4MHD9b+/ft14MABhYWFScrda6yU/bU5N/79GSctLU21atXSG2+8ka1vTjerytrnp06dqgoVKjhNuziQtW7dWkuWLFHfvn21dOlSRUZGqnTp0pIuXC/Wr18/denSRUOGDFGpUqW0ZcsWjR49WufPn3fUeLmxl5v34bZt2+qNN97Qtm3btG3bNlWuXDnXP01Q0HBziAKiWrVqKlKkiDZs2JBtmp+fn+NoS5atW7fK09NTt99+e67XUahQIceLSpaaNWvqjz/+0G233ZbtX9bA27p1q6ZPn64PPvhAxYoV0/jx4y2XayUzM1O//fab4/G+fft0+vRpxzdJOalVq5aOHz8uLy+vbLVmvaj4+/srLi4u13UUKVJEUVFRev755zVr1ixt27Ytx6MNfn5++vPPP51uYPHHH39Y1oz8daONm38LDAyUzWbTiRMnstWR9Q1sQEDAJcfEyZMntX//fg0YMEANGjSQn5+f46YRuH5dyd8960OqzWb7z+vPer1OTk52tFn9XpSVcuXKqXz58jp06FC2sVClShVHvxIlSqhFixaaMGGC3nrrLa1atUp///23pNy/HyB/5dXrUsWKFdWlSxe9++676tmzp+bPn++YFhYWpo0bN2rTpk0KDw/XzTffLD8/P3344Yfy9fV1vEfk5jU2r9SqVUsHDx5U2bJls60r68vhi/n5+cnHx0dHjx7N1v/iL/BatWql33//Xb/99ptWrVqlNm3aOKbt2LFDxhiNHDlSISEhuv3227Pd9MFKiRIlVKlSpct+9ipdurSaNWumRYsWKTY2Vu3bt3dpHQUJwamAKFy4sPr06aPXX39dixcvVmJion755RctWLBArVu3lo+Pj0aOHKm9e/dqw4YNGj9+vNq2bes43Sg3KlWqpE2bNikpKUknTpyQJPXp00fbtm3TuHHjtGvXLh04cEDffPON424oqampGj58uLp166bGjRvrjTfe0PLlyx13bbnUcq0UKlRI48eP16+//qrffvtNzz33nEJCQhynvOUkIiJCISEhGjhwoNavX6/Dhw9r69ateuuttxQfHy9JGjRokJYtW6bJkycrISFBe/bscRx+/7dFixZpwYIF2rt3rw4dOqSvvvpKRYoU0a233prjuv39/fXss89qx44d2r59u4YPH67w8HCnUw5xdd1o4+bfbr/9drVu3VrDhw/X6tWrdejQIW3fvl1Tp07V2rVrJUl9+/ZVfHy8XnrpJe3evVsJCQmaM2eOTpw4oVKlSunmm2/WvHnzdPDgQcXFxemVV165olpw7biSv3ulSpXk4eGhtWvX6sSJE0533XJVkSJFFBISomnTpikhIUE///yz3n777SteXpann35a06ZN06xZs7R//37t2bNHCxcuVExMjKQLdw9bunSpEhIStH//fq1cuVK+vr4qWbKkS+8HyF958bo0ceJE/fDDDzp06JB27NihjRs3On3JGR4ervXr18vb29vRHh4eriVLlig8PNzRLzevsXmldevWKl26tAYMGKDNmzfr0KFD2rhxoyZMmKA///wzW/8SJUqoV69eio6OVmxsrBITE7Vjxw7Nnj1bsbGxjn6VK1dW3bp1NXr0aNlsNkVFRTmm3XbbbTp//rxmz56tQ4cOafHixZo7d67LtT/11FOKiYnRrFmzdODAAUcdF+vYsaNiY2OVkJCghx56yOV1FBQEpwLkySefVM+ePTV58mS1aNFCQ4YM0YkTJ1S0aFHNmDFDf//9tx5++GE988wzatCggdN53Lnx9NNP68iRI2rWrJnjXNQaNWpo9uzZOnDggB599FG1a9dOkydPVvny5SVdePEpWrSohg4dKunCt9dDhw7VmDFjlJSUdMnlWilSpIj69OmjYcOGqUuXLipWrJjeeuuty87j4eGhadOmKSwsTM8995yaN2+uoUOH6siRI44PwvXr19c777yj7777Tm3bttXjjz/uCFX/VrJkSS1YsEBdunRRmzZtFBcXpw8//NBxCPvf637//fdVsmRJde3aVT169FCVKlUsa0b+u5HGTU6io6P10EMP6ZVXXtGDDz6oJ598UvHx8Y5vHG+//XZ9/PHH2r17tzp27KhHHnlE3377rby9veXp6am33npLO3bsUKtWrRQdHa3hw4dfcS24NlzJ371ChQp66qmnNGnSJEVERGQ7guqql19+WTabTe3bt9fLL7+swYMH/6flSRc+mE2YMEGLFi1S69at1a1bN8XGxqpy5cqSpOLFi2v69Ol6+OGH9fDDD+vIkSOaNm2aPD09XXo/QP7Ki9clu92ucePGqUWLFnriiSdUrVo1jRkzxjG9Xr16stvtjlPypAufH2w2m1NwkqxfY/NK0aJF9emnn+rWW2/VoEGD1KJFC40ePVrnzp275GnegwcP1pNPPqmpU6c6nuvatWsd+3yW1q1ba/fu3brvvvucTt+uUaOGnnvuOX300Udq1aqVlixZ4njfckW7du00atQozZkzR61atVK/fv0ct3PPEhERofLlyysyMjLbqYXXEg/DBRi4yhYtWqSXX35ZmzdvdncpAAAAyGdnzpxRo0aNFB0drfvvv9/d5Vwxbg4BAAAAIM/Z7XadPHlSH3/8sUqWLOl0quC1iOCEPPfEE09oy5YtOU7r16+f43QmAP/Hatz079//KlcEALgSdevWveS0jz766Jq9o9yVOHr0qO69917dcssteuWVV675W/xzqh7yXFJSks6ePZvjtKyLPgE4Y9wAwPXh39f3XKxChQou/XQKChaCEwAAAABY4K56AAAAAGCB4AQAAAAAFghOAAAAAGCB4AQAAAAAFghOAADkQkBAgL755ht3lwEAcBOCEwDgmjFy5EgFBAToxRdfzDZt7NixCggI0MiRI3O1rI0bNyogIECnT5/OVf/169erUaNGLtULALh+EJwAANeUihUravny5U6/e3Xu3DktXbpUt956a56vLyMjQ5Lk6+srHx+fPF8+AODaQHACAFxTatasqYoVK2r16tWOttWrV6tixYoKDAx0tNntdk2dOlVRUVEKDg5WmzZttHLlSknS4cOH1b17d0lSWFiY05Gqbt26ady4cZo4caLq16+v3r17S8p+qt6ff/6poUOHKjw8XCEhIWrfvr1+/fXXfH/+AAD38HZ3AQAAuKpDhw5atGiR2rRpI0lauHCh2rdvr59//tnRZ+rUqfrqq680duxYVatWTZs2bdL//vc/lSlTRnfddZemTJmip556SitXrlSJEiVUpEgRx7yxsbHq0qWLPv/88xzXf+bMGXXt2lUVKlTQ+++/L19fX+3YsUN2uz1/nzgAwG0ITgCAa06bNm00adIkHTlyRJK0detWvfnmm47glJGRoalTpyomJkZ169aVJFWpUkVbtmzRvHnzFB4erlKlSkmSypYtq5IlSzotv1q1aho+fPgl17906VKdOHFCX3zxhW6++WZJ0m233ZbXTxMAUIAQnAAA15wyZcqoSZMmio2NlTFGTZo0UZkyZRzTDx48qPT0dPXq1ctpvvPnzzudzncptWrVuuz0Xbt2qWbNmo7QBAC4/hGcAADXpA4dOmjcuHGSpDFjxjhNS0tLk3ThdL0KFSo4TcvNDR6KFi162ekXn9YHALgxEJwAANekhg0b6vz58/Lw8FBkZKTTND8/P/n4+Ojo0aMKDw/Pcf5ChQpJkmw2m8vrDggI0IIFC/T3339z1AkAbhAEJwDANcnLy0srVqxw/P/FSpQooV69eik6OlrGGN111136559/tHXrVpUoUULt2rVTpUqV5OHhobVr16px48YqXLiwihcvnqt1t2zZUh9++KEGDhyooUOHqnz58tq5c6fKly/vuKYKAHB94XbkAIBrVokSJVSiRIkcpw0ePFhPPvmkpk6dqhYtWuiJJ57Q2rVrVblyZUlShQoV9NRTT2nSpEmKiIjQ+PHjc71eHx8fffzxxypbtqz69u2r1q1ba9q0adkCHADg+uFhjDHuLgIAAAAACjKOOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACAhf8HRJ6WlUx3T2sAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def plot_average_metrics(metrics):\n", + " avg_metrics = {metric: data[metric].mean() for metric in metrics}\n", + "\n", + " avg_metrics_df = pd.DataFrame({\n", + " 'Metric': list(avg_metrics.keys()),\n", + " 'Average Score': list(avg_metrics.values())\n", + " })\n", + "\n", + " plt.figure(figsize=(10, 5))\n", + " sns.barplot(x='Metric', y='Average Score', data=avg_metrics_df, palette='viridis', hue='Metric', dodge=False)\n", + " plt.ylim(0, 1.1)\n", + " plt.title('Average Score of Each Metric')\n", + " plt.ylabel('Average Score')\n", + " plt.xlabel('Metric')\n", + " plt.legend([],[], frameon=False)\n", + " plt.show()\n", + "\n", + "metrics = ['context_precision', 'context_recall', 'faithfulness', 'answer_relevancy']\n", + "plot_average_metrics(metrics)\n" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "\n", + "- **Queries with Low Context Precision (<0.7):**\n", + " - Improve retrieval mechanisms to filter out irrelevant contexts.\n", + "\n", + "- **Queries with Low Context Recall (<0.7):**\n", + " - Enhance the retrieval system to capture more relevant contexts.\n", + "\n", + "- **Queries with Low Faithfulness (<0.9):**\n", + " - Review the generated answers for unsupported claims.\n", + "\n", + "- **Queries with Low Answer Relevancy (<0.7):**\n", + " - Ensure the response generation effectively utilizes the user query and retrieved contexts." + ] + }, + { + "cell_type": "markdown", + "id": "17", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "18", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
user_inputcontext_precisioncontext_recallfaithfulnessanswer_relevancy
1How many days of paid sick leave do most full-time employees earn per year under Civil Service Rules?1.0000000.5000001.0000000.986973
2What are the three categories of employees eligible for health coverage?0.7000001.0000000.8000000.981254
3How long must an employee wait before using vacation time after starting employment?0.4166670.6666671.0000000.979251
6How long is the normal probationary period for permanent civil service positions?1.0000000.6666671.0000000.000000
7What are employees required to do in case of a catastrophic event while off duty?0.8666670.6666671.0000000.987241
8What is the city's policy on accepting gifts from subordinates?0.8875001.0000000.6666670.956919
\n", + "
" + ], + "text/plain": [ + " user_input \\\n", + "1 How many days of paid sick leave do most full-time employees earn per year under Civil Service Rules? \n", + "2 What are the three categories of employees eligible for health coverage? \n", + "3 How long must an employee wait before using vacation time after starting employment? \n", + "6 How long is the normal probationary period for permanent civil service positions? \n", + "7 What are employees required to do in case of a catastrophic event while off duty? \n", + "8 What is the city's policy on accepting gifts from subordinates? \n", + "\n", + " context_precision context_recall faithfulness answer_relevancy \n", + "1 1.000000 0.500000 1.000000 0.986973 \n", + "2 0.700000 1.000000 0.800000 0.981254 \n", + "3 0.416667 0.666667 1.000000 0.979251 \n", + "6 1.000000 0.666667 1.000000 0.000000 \n", + "7 0.866667 0.666667 1.000000 0.987241 \n", + "8 0.887500 1.000000 0.666667 0.956919 " + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "low_metrics = data[\n", + " (data['context_precision'] < 0.7) |\n", + " (data['context_recall'] < 0.7) |\n", + " (data['faithfulness'] < 0.9) |\n", + " (data['answer_relevancy'] < 0.7)\n", + "]\n", + "\n", + "low_metrics[['user_input', 'context_precision', 'context_recall', 'faithfulness', 'answer_relevancy']]" + ] + }, + { + "cell_type": "markdown", + "id": "19", + "metadata": {}, + "source": [ + "---" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/DocQA/docker/compose-cpu.yaml b/examples/DocQA/docker/compose-cpu.yaml new file mode 100644 index 000000000..9b9ea0bad --- /dev/null +++ b/examples/DocQA/docker/compose-cpu.yaml @@ -0,0 +1,53 @@ +services: + chromadb: + image: ghcr.io/chroma-core/chroma:latest + env_file: "docqa_env" + environment: + - IS_PERSISTENT=TRUE + - CHROMA_HOST_PORT=${CHROMA_PORT} + network_mode: "host" + volumes: + - ${DOC_PATH}:/chroma/chroma + tty: true + ports: + - "6000:6000" + ollama: + image: ollama/ollama:latest + env_file: "docqa_env" + network_mode: "host" + environment: + - MODEL_NAME=${MODEL_NAME} + - OLLAMA_KEEP_ALIVE=99h + - OLLAMA_HOST=127.0.0.1:14343 + volumes: + - ollama:/root/.ollama # this solution synchronizes with the docker volume and loads the model rocket fast + - ./ollama_start.sh:/root/ollama_start.sh + ports: + - "14343:14343" + tty: true + restart: always + entrypoint: ["bash", "/root/ollama_start.sh"] + llamastack: + image: llamastack/distribution-ollama:0.0.56 + env_file: "docqa_env" + network_mode: "host" + tty: true + volumes: + - ~/.llama:/root/.llama + # Link to ollama run.yaml file + - ./llama_stack_run.yaml:/root/my-run.yaml + - ../../DocQA:/root/DocQA + - ${DOC_PATH}:/root/rag_data/ + - ./llama_stack_start.sh:/root/llama_stack_start.sh + ports: + - "5000:5000" # for llama-stack + - "7860:7860" # for UI + # Hack: wait for ollama server to start before starting docker + entrypoint: ["bash", "/root/llama_stack_start.sh"] + #entrypoint: bash -c "sleep 60; python -m llama_stack.distribution.server.server --yaml_config /root/my-run.yaml" + deploy: + restart_policy: + condition: "no" + +volumes: + ollama: diff --git a/examples/DocQA/docker/compose-gpu.yaml b/examples/DocQA/docker/compose-gpu.yaml new file mode 100644 index 000000000..f7a5b6fde --- /dev/null +++ b/examples/DocQA/docker/compose-gpu.yaml @@ -0,0 +1,59 @@ +services: + chromadb: + image: ghcr.io/chroma-core/chroma:latest + env_file: "docqa_env" + environment: + - IS_PERSISTENT=TRUE + - CHROMA_HOST_PORT=${CHROMA_PORT} + network_mode: "host" + volumes: + - ${DOC_PATH}:/chroma/chroma + tty: true + ports: + - "6000:6000" + ollama: + image: ollama/ollama:latest + env_file: "docqa_env" + network_mode: "host" + environment: + - MODEL_NAME=${MODEL_NAME} + - OLLAMA_KEEP_ALIVE=99h + - OLLAMA_HOST=127.0.0.1:14343 + volumes: + - ollama:/root/.ollama # this solution synchronizes with the docker volume and loads the model rocket fast + - ./ollama_start.sh:/root/ollama_start.sh + ports: + - "14343:14343" + tty: true + deploy: + resources: # enable GPUs + reservations: + devices: + - driver: nvidia + capabilities: [gpu] + restart: always + entrypoint: ["bash", "/root/ollama_start.sh"] + llamastack: + image: llamastack/distribution-ollama:0.0.56 + env_file: "docqa_env" + network_mode: "host" + tty: true + volumes: + - ~/.llama:/root/.llama + # Link to ollama run.yaml file + - ./llama_stack_run.yaml:/root/my-run.yaml + - ../../DocQA:/root/DocQA + - ${DOC_PATH}:/root/rag_data/ + - ./llama_stack_start.sh:/root/llama_stack_start.sh + ports: + - "5000:5000" # for llama-stack + - "7860:7860" # for UI + # Hack: wait for ollama server to start before starting docker + entrypoint: ["bash", "/root/llama_stack_start.sh"] + #entrypoint: bash -c "sleep 60; python -m llama_stack.distribution.server.server --yaml_config /root/my-run.yaml" + deploy: + restart_policy: + condition: "no" + +volumes: + ollama: diff --git a/examples/DocQA/docker/docqa_env_template b/examples/DocQA/docker/docqa_env_template new file mode 100644 index 000000000..2a45408aa --- /dev/null +++ b/examples/DocQA/docker/docqa_env_template @@ -0,0 +1,7 @@ +DOC_PATH=/path/to/your/llama-stack-apps/examples/DocQA/example_data +MODEL_NAME=llama3.2:1b-instruct-fp16 +HOST=localhost +LLAMA_STACK_PORT=5000 +CHROMA_PORT=6000 +GRADIO_SERVER_PORT=7860 +USE_GPU_FOR_DOC_INGESTION=false diff --git a/examples/DocQA/docker/llama_stack_run.yaml b/examples/DocQA/docker/llama_stack_run.yaml new file mode 100644 index 000000000..3d67c8200 --- /dev/null +++ b/examples/DocQA/docker/llama_stack_run.yaml @@ -0,0 +1,54 @@ +version: '2' +built_at: '2024-10-08T17:40:45.325529' +image_name: local +docker_image: null +conda_env: local +apis: +- shields +- agents +- models +- memory +- memory_banks +- inference +- safety +providers: + inference: + - provider_id: remote::ollama + provider_type: remote::ollama + config: + url: http://127.0.0.1:14343 + memory: + - provider_id: remote::chromadb + provider_type: remote::chromadb + config: + host: localhost + port: 6000 + protocol: http + safety: + - provider_id: inline::llama-guard-0 + provider_type: inline::llama-guard + config: + excluded_categories: [] + agents: + - provider_id: inline::meta-reference-0 + provider_type: inline::meta-reference + config: + persistence_store: + namespace: null + type: sqlite + db_path: ${HOME}/.llama/runtime/kvstore.db + telemetry: + - provider_id: inline::meta-reference-0 + provider_type: inline::meta-reference + config: {} +metadata_store: null +models: +- metadata: {} + model_id: meta-llama/Llama-3.2-1B-Instruct + provider_id: null + provider_model_id: llama3.2:1b-instruct-fp16 +shields: [] +memory_banks: [] +datasets: [] +scoring_fns: [] +eval_tasks: [] diff --git a/examples/DocQA/docker/llama_stack_start.sh b/examples/DocQA/docker/llama_stack_start.sh new file mode 100644 index 000000000..a7af16be7 --- /dev/null +++ b/examples/DocQA/docker/llama_stack_start.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +sleep 45 +echo "-----starting to llama-stack docker now---------" +pip install gradio + +if [ "$USE_GPU_FOR_DOC_INGESTION" = true ]; then + echo "Using GPU to ingest files" + pip install docling + python /root/DocQA/scripts/ingest_files.py --input_dir /root/rag_data/ +fi +echo "starting the llama-stack server" +python -m llama_stack.distribution.server.server --yaml-config /root/my-run.yaml --disable-ipv6& +sleep 30 +echo "---------running the RAG app--------------" +python /root/DocQA/app.py diff --git a/examples/DocQA/docker/ollama_start.sh b/examples/DocQA/docker/ollama_start.sh new file mode 100644 index 000000000..d38a90c79 --- /dev/null +++ b/examples/DocQA/docker/ollama_start.sh @@ -0,0 +1,6 @@ +#!/bin/bash +echo "-------------start to serve------------" +OLLAMA_HOST=127.0.0.1:14343 /usr/bin/ollama serve & +echo "Running ollama model: $MODEL_NAME" +sleep 5 +OLLAMA_HOST=127.0.0.1:14343 /usr/bin/ollama run $MODEL_NAME diff --git a/examples/DocQA/docker/run_RAG.sh b/examples/DocQA/docker/run_RAG.sh new file mode 100755 index 000000000..3380fbe6a --- /dev/null +++ b/examples/DocQA/docker/run_RAG.sh @@ -0,0 +1,12 @@ +set -a +source docqa_env +echo $DOC_PATH +# Run GPU version of ollama docker +if [ "$USE_GPU_FOR_DOC_INGESTION" = true ]; then + echo "Running with GPU" + docker compose --file compose-gpu.yaml up +else +# Run CPU version of ollama docker + echo "Running with CPU only" + docker compose --file compose-cpu.yaml up +fi diff --git a/examples/DocQA/example_data/llama_3.1.md b/examples/DocQA/example_data/llama_3.1.md new file mode 100644 index 000000000..af2d4850b --- /dev/null +++ b/examples/DocQA/example_data/llama_3.1.md @@ -0,0 +1,977 @@ +## Model Information + +The Meta Llama 3.1 collection of multilingual large language models (LLMs) is a collection of pretrained and instruction tuned generative models in 8B, 70B and 405B sizes (text in/text out). The Llama 3.1 instruction tuned text only models (8B, 70B, 405B) are optimized for multilingual dialogue use cases and outperform many of the available open source and closed chat models on common industry benchmarks. + +Model developer: Meta + +Model Architecture: Llama 3.1 is an auto-regressive language model that uses an optimized transformer architecture. The tuned versions use supervised fine-tuning (SFT) and reinforcement learning with human feedback (RLHF) to align with human preferences for helpfulness and safety. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Training Data + Params + Input modalities + Output modalities + Context length + GQA + Token count + Knowledge cutoff +
Llama 3.1 (text only) + A new mix of publicly available online data. + 8B + Multilingual Text + Multilingual Text and code + 128k + Yes + 15T+ + December 2023 +
70B + Multilingual Text + Multilingual Text and code + 128k + Yes +
405B + Multilingual Text + Multilingual Text and code + 128k + Yes +
+ + +Supported languages: English, German, French, Italian, Portuguese, Hindi, Spanish, and Thai. + +Llama 3.1 family of models. Token counts refer to pretraining data only. All model versions use Grouped-Query Attention (GQA) for improved inference scalability. + +Model Release Date: July 23, 2024. + +Status: This is a static model trained on an offline dataset. Future versions of the tuned models will be released as we improve model safety with community feedback. + +License: A custom commercial license, the Llama 3.1 Community License, is available at: [https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE) + +Feedback: Instructions on how to provide feedback or comments on the model can be found in the Llama Models [README](https://github.com/meta-llama/llama-models/blob/main/README.md). For more technical information about generation parameters and recipes for how to use Llama 3.1 in applications, please go [here](https://github.com/meta-llama/llama-recipes). + + +## Intended Use + +Intended Use Cases Llama 3.1 is intended for commercial and research use in multiple languages. Instruction tuned text only models are intended for assistant-like chat, whereas pretrained models can be adapted for a variety of natural language generation tasks. The Llama 3.1 model collection also supports the ability to leverage the outputs of its models to improve other models including synthetic data generation and distillation. The Llama 3.1 Community License allows for these use cases. + +Out-of-scope Use in any manner that violates applicable laws or regulations (including trade compliance laws). Use in any other way that is prohibited by the Acceptable Use Policy and Llama 3.1 Community License. Use in languages beyond those explicitly referenced as supported in this model card. + +Note: Llama 3.1 has been trained on a broader collection of languages than the 8 supported languages. Developers may fine-tune Llama 3.1 models for languages beyond the 8 supported languages provided they comply with the Llama 3.1 Community License and the Acceptable Use Policy and in such cases are responsible for ensuring that any uses of Llama 3.1 in additional languages is done in a safe and responsible manner. + + +## Hardware and Software + +Training Factors We used custom training libraries, Meta's custom built GPU cluster, and production infrastructure for pretraining. Fine-tuning, annotation, and evaluation were also performed on production infrastructure. + +Training Energy Use Training utilized a cumulative of 39.3M GPU hours of computation on H100-80GB (TDP of 700W) type hardware, per the table below. Training time is the total GPU time required for training each model and power consumption is the peak power capacity per GPU device used, adjusted for power usage efficiency. + + +Training Greenhouse Gas Emissions Estimated total location-based greenhouse gas emissions were 11,390 tons CO2eq for training. Since 2020, Meta has maintained net zero greenhouse gas emissions in its global operations and matched 100% of its electricity use with renewable energy, therefore the total market-based greenhouse gas emissions for training were 0 tons CO2eq. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Training Time (GPU hours) + Training Power Consumption (W) + Training Location-Based Greenhouse Gas Emissions +

+(tons CO2eq) +

Training Market-Based Greenhouse Gas Emissions +

+(tons CO2eq) +

Llama 3.1 8B + 1.46M + 700 + 420 + 0 +
Llama 3.1 70B + 7.0M + 700 + 2,040 + 0 +
Llama 3.1 405B + 30.84M + 700 + 8,930 + 0 +
Total + 39.3M + +
    + +
+ 		11,390 + 0 +
+ + + +The methodology used to determine training energy use and greenhouse gas emissions can be found [here](https://arxiv.org/pdf/2204.05149). Since Meta is openly releasing these models, the training energy use and greenhouse gas emissions will not be incurred by others. + + +## Training Data + +Overview: Llama 3.1 was pretrained on ~15 trillion tokens of data from publicly available sources. The fine-tuning data includes publicly available instruction datasets, as well as over 25M synthetically generated examples. + +Data Freshness: The pretraining data has a cutoff of December 2023. + + +## Benchmark scores + +In this section, we report the results for Llama 3.1 models on standard automatic benchmarks. For all the evaluations, we use our internal evaluations library. Details of our evals can be found [here](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/eval_details.md). We are also releasing the raw data generated as part of our evals which can be found [here](https://huggingface.co/meta-llama) in the dataset sections. + +### Base pretrained models + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Category + Benchmark + # Shots + Metric + Llama 3 8B + Llama 3.1 8B + Llama 3 70B + Llama 3.1 70B + Llama 3.1 405B +
General + MMLU + 5 + macro_avg/acc_char + 66.7 + 66.7 + 79.5 + 79.3 + 85.2 +
MMLU-Pro (CoT) + 5 + macro_avg/acc_char + 36.2 + 37.1 + 55.0 + 53.8 + 61.6 +
AGIEval English + 3-5 + average/acc_char + 47.1 + 47.8 + 63.0 + 64.6 + 71.6 +
CommonSenseQA + 7 + acc_char + 72.6 + 75.0 + 83.8 + 84.1 + 85.8 +
Winogrande + 5 + acc_char + - + 60.5 + - + 83.3 + 86.7 +
BIG-Bench Hard (CoT) + 3 + average/em + 61.1 + 64.2 + 81.3 + 81.6 + 85.9 +
ARC-Challenge + 25 + acc_char + 79.4 + 79.7 + 93.1 + 92.9 + 96.1 +
Knowledge reasoning + TriviaQA-Wiki + 5 + em + 78.5 + 77.6 + 89.7 + 89.8 + 91.8 +
Reading comprehension + SQuAD + 1 + em + 76.4 + 77.0 + 85.6 + 81.8 + 89.3 +
QuAC (F1) + 1 + f1 + 44.4 + 44.9 + 51.1 + 51.1 + 53.6 +
BoolQ + 0 + acc_char + 75.7 + 75.0 + 79.0 + 79.4 + 80.0 +
DROP (F1) + 3 + f1 + 58.4 + 59.5 + 79.7 + 79.6 + 84.8 +
+ + + +### Instruction tuned models + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Category + Benchmark + # Shots + Metric + Llama 3 8B Instruct + Llama 3.1 8B Instruct + Llama 3 70B Instruct + Llama 3.1 70B Instruct + Llama 3.1 405B Instruct +
General + MMLU + 5 + macro_avg/acc + 68.5 + 69.4 + 82.0 + 83.6 + 87.3 +
MMLU (CoT) + 0 + macro_avg/acc + 65.3 + 73.0 + 80.9 + 86.0 + 88.6 +
MMLU-Pro (CoT) + 5 + macro_avg/acc + 45.5 + 48.3 + 63.4 + 66.4 + 73.3 +
IFEval + + + 76.8 + 80.4 + 82.9 + 87.5 + 88.6 +
Reasoning + ARC-C + 0 + acc + 82.4 + 83.4 + 94.4 + 94.8 + 96.9 +
GPQA + 0 + em + 34.6 + 30.4 + 39.5 + 46.7 + 50.7 +
Code + HumanEval + 0 + pass@1 + 60.4 + 72.6 + 81.7 + 80.5 + 89.0 +
MBPP ++ base version + 0 + pass@1 + 70.6 + 72.8 + 82.5 + 86.0 + 88.6 +
Multipl-E HumanEval + 0 + pass@1 + - + 50.8 + - + 65.5 + 75.2 +
Multipl-E MBPP + 0 + pass@1 + - + 52.4 + - + 62.0 + 65.7 +
Math + GSM-8K (CoT) + 8 + em_maj1@1 + 80.6 + 84.5 + 93.0 + 95.1 + 96.8 +
MATH (CoT) + 0 + final_em + 29.1 + 51.9 + 51.0 + 68.0 + 73.8 +
Tool Use + API-Bank + 0 + acc + 48.3 + 82.6 + 85.1 + 90.0 + 92.0 +
BFCL + 0 + acc + 60.3 + 76.1 + 83.0 + 84.8 + 88.5 +
Gorilla Benchmark API Bench + 0 + acc + 1.7 + 8.2 + 14.7 + 29.7 + 35.3 +
Nexus (0-shot) + 0 + macro_avg/acc + 18.1 + 38.5 + 47.8 + 56.7 + 58.7 +
Multilingual + Multilingual MGSM (CoT) + 0 + em + - + 68.9 + - + 86.9 + 91.6 +
+ +#### Multilingual benchmarks + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Category + Benchmark + Language + Llama 3.1 8B Instruct + Llama 3.1 70B Instruct + Llama 3.1 405B Instruct +
General + MMLU (5-shot, macro_avg/acc) + Portuguese + 62.12 + 80.13 + 84.95 +
Spanish + 62.45 + 80.05 + 85.08 +
Italian + 61.63 + 80.4 + 85.04 +
German + 60.59 + 79.27 + 84.36 +
French + 62.34 + 79.82 + 84.66 +
Hindi + 50.88 + 74.52 + 80.31 +
Thai + 50.32 + 72.95 + 78.21 +
+ + + +## Responsibility & Safety + +As part of our Responsible release approach, we followed a three-pronged strategy to managing trust & safety risks: + + + +* Enable developers to deploy helpful, safe and flexible experiences for their target audience and for the use cases supported by Llama. +* Protect developers against adversarial users aiming to exploit Llama capabilities to potentially cause harm. +* Provide protections for the community to help prevent the misuse of our models. + + +### Responsible deployment + +Llama is a foundational technology designed to be used in a variety of use cases, examples on how Meta’s Llama models have been responsibly deployed can be found in our [Community Stories webpage](https://llama.meta.com/community-stories/). Our approach is to build the most helpful models enabling the world to benefit from the technology power, by aligning our model safety for the generic use cases addressing a standard set of harms. Developers are then in the driver seat to tailor safety for their use case, defining their own policy and deploying the models with the necessary safeguards in their Llama systems. Llama 3.1 was developed following the best practices outlined in our Responsible Use Guide, you can refer to the [Responsible Use Guide](https://llama.meta.com/responsible-use-guide/) to learn more. + + +#### Llama 3.1 instruct + +Our main objectives for conducting safety fine-tuning are to provide the research community with a valuable resource for studying the robustness of safety fine-tuning, as well as to offer developers a readily available, safe, and powerful model for various applications to reduce the developer workload to deploy safe AI systems. For more details on the safety mitigations implemented please read the Llama 3 paper. + +Fine-tuning data + +We employ a multi-faceted approach to data collection, combining human-generated data from our vendors with synthetic data to mitigate potential safety risks. We’ve developed many large language model (LLM)-based classifiers that enable us to thoughtfully select high-quality prompts and responses, enhancing data quality control. + +Refusals and Tone + +Building on the work we started with Llama 3, we put a great emphasis on model refusals to benign prompts as well as refusal tone. We included both borderline and adversarial prompts in our safety data strategy, and modified our safety data responses to follow tone guidelines. + + +#### Llama 3.1 systems + +Large language models, including Llama 3.1, are not designed to be deployed in isolation but instead should be deployed as part of an overall AI system with additional safety guardrails as required. Developers are expected to deploy system safeguards when building agentic systems. Safeguards are key to achieve the right helpfulness-safety alignment as well as mitigating safety and security risks inherent to the system and any integration of the model or system with external tools. + +As part of our responsible release approach, we provide the community with [safeguards](https://llama.meta.com/trust-and-safety/) that developers should deploy with Llama models or other LLMs, including Llama Guard 3, Prompt Guard and Code Shield. All our [reference implementations](https://github.com/meta-llama/llama-agentic-system) demos contain these safeguards by default so developers can benefit from system-level safety out-of-the-box. + + +#### New capabilities + +Note that this release introduces new capabilities, including a longer context window, multilingual inputs and outputs and possible integrations by developers with third party tools. Building with these new capabilities requires specific considerations in addition to the best practices that generally apply across all Generative AI use cases. + +Tool-use: Just like in standard software development, developers are responsible for the integration of the LLM with the tools and services of their choice. They should define a clear policy for their use case and assess the integrity of the third party services they use to be aware of the safety and security limitations when using this capability. Refer to the Responsible Use Guide for best practices on the safe deployment of the third party safeguards. + +Multilinguality: Llama 3.1 supports 7 languages in addition to English: French, German, Hindi, Italian, Portuguese, Spanish, and Thai. Llama may be able to output text in other languages than those that meet performance thresholds for safety and helpfulness. We strongly discourage developers from using this model to converse in non-supported languages without implementing finetuning and system controls in alignment with their policies and the best practices shared in the Responsible Use Guide. + + +### Evaluations + +We evaluated Llama models for common use cases as well as specific capabilities. Common use cases evaluations measure safety risks of systems for most commonly built applications including chat bot, coding assistant, tool calls. We built dedicated, adversarial evaluation datasets and evaluated systems composed of Llama models and Llama Guard 3 to filter input prompt and output response. It is important to evaluate applications in context, and we recommend building dedicated evaluation dataset for your use case. Prompt Guard and Code Shield are also available if relevant to the application. + +Capability evaluations measure vulnerabilities of Llama models inherent to specific capabilities, for which were crafted dedicated benchmarks including long context, multilingual, tools calls, coding or memorization. + +Red teaming + +For both scenarios, we conducted recurring red teaming exercises with the goal of discovering risks via adversarial prompting and we used the learnings to improve our benchmarks and safety tuning datasets. + +We partnered early with subject-matter experts in critical risk areas to understand the nature of these real-world harms and how such models may lead to unintended harm for society. Based on these conversations, we derived a set of adversarial goals for the red team to attempt to achieve, such as extracting harmful information or reprogramming the model to act in a potentially harmful capacity. The red team consisted of experts in cybersecurity, adversarial machine learning, responsible AI, and integrity in addition to multilingual content specialists with background in integrity issues in specific geographic markets. . + + +### Critical and other risks + +We specifically focused our efforts on mitigating the following critical risk areas: + +1. CBRNE (Chemical, Biological, Radiological, Nuclear, and Explosive materials) helpfulness + +To assess risks related to proliferation of chemical and biological weapons, we performed uplift testing designed to assess whether use of Llama 3.1 models could meaningfully increase the capabilities of malicious actors to plan or carry out attacks using these types of weapons. + + +2. Child Safety + +Child Safety risk assessments were conducted using a team of experts, to assess the model’s capability to produce outputs that could result in Child Safety risks and inform on any necessary and appropriate risk mitigations via fine tuning. We leveraged those expert red teaming sessions to expand the coverage of our evaluation benchmarks through Llama 3 model development. For Llama 3, we conducted new in-depth sessions using objective based methodologies to assess the model risks along multiple attack vectors including the additional languages Llama 3 is trained on. We also partnered with content specialists to perform red teaming exercises assessing potentially violating content while taking account of market specific nuances or experiences. + +3. Cyber attack enablement + +Our cyber attack uplift study investigated whether LLMs can enhance human capabilities in hacking tasks, both in terms of skill level and speed. + +Our attack automation study focused on evaluating the capabilities of LLMs when used as autonomous agents in cyber offensive operations, specifically in the context of ransomware attacks. This evaluation was distinct from previous studies that considered LLMs as interactive assistants. The primary objective was to assess whether these models could effectively function as independent agents in executing complex cyber-attacks without human intervention. + +Our study of Llama 3.1 405B’s social engineering uplift for cyber attackers was conducted to assess the effectiveness of AI models in aiding cyber threat actors in spear phishing campaigns. Please read our Llama 3.1 Cyber security whitepaper to learn more. + + +### Community + +Generative AI safety requires expertise and tooling, and we believe in the strength of the open community to accelerate its progress. We are active members of open consortiums, including the AI Alliance, Partnership on AI and MLCommons, actively contributing to safety standardization and transparency. We encourage the community to adopt taxonomies like the MLCommons Proof of Concept evaluation to facilitate collaboration and transparency on safety and content evaluations. Our Purple Llama tools are open sourced for the community to use and widely distributed across ecosystem partners including cloud service providers. We encourage community contributions to our [Github repository](https://github.com/meta-llama/PurpleLlama). + +We also set up the [Llama Impact Grants](https://llama.meta.com/llama-impact-grants/) program to identify and support the most compelling applications of Meta’s Llama model for societal benefit across three categories: education, climate and open innovation. The 20 finalists from the hundreds of applications can be found [here](https://llama.meta.com/llama-impact-grants/#finalists). + +Finally, we put in place a set of resources including an [output reporting mechanism](https://developers.facebook.com/llama_output_feedback) and [bug bounty program](https://www.facebook.com/whitehat) to continuously improve the Llama technology with the help of the community. + + +## Ethical Considerations and Limitations + +The core values of Llama 3.1 are openness, inclusivity and helpfulness. It is meant to serve everyone, and to work for a wide range of use cases. It is thus designed to be accessible to people across many different backgrounds, experiences and perspectives. Llama 3.1 addresses users and their needs as they are, without insertion unnecessary judgment or normativity, while reflecting the understanding that even content that may appear problematic in some cases can serve valuable purposes in others. It respects the dignity and autonomy of all users, especially in terms of the values of free thought and expression that power innovation and progress. + +But Llama 3.1 is a new technology, and like any new technology, there are risks associated with its use. Testing conducted to date has not covered, nor could it cover, all scenarios. For these reasons, as with all LLMs, Llama 3.1’s potential outputs cannot be predicted in advance, and the model may in some instances produce inaccurate, biased or other objectionable responses to user prompts. Therefore, before deploying any applications of Llama 3.1 models, developers should perform safety testing and tuning tailored to their specific applications of the model. Please refer to available resources including our [Responsible Use Guide](https://llama.meta.com/responsible-use-guide), [Trust and Safety](https://llama.meta.com/trust-and-safety/) solutions, and other [resources](https://llama.meta.com/docs/get-started/) to learn more about responsible development. diff --git a/examples/DocQA/example_data/llama_3.2.md b/examples/DocQA/example_data/llama_3.2.md new file mode 100644 index 000000000..0bc604261 --- /dev/null +++ b/examples/DocQA/example_data/llama_3.2.md @@ -0,0 +1,216 @@ +## Model Information + +The Llama 3.2 collection of multilingual large language models (LLMs) is a collection of pretrained and instruction-tuned generative models in 1B and 3B sizes (text in/text out). The Llama 3.2 instruction-tuned text only models are optimized for multilingual dialogue use cases, including agentic retrieval and summarization tasks. They outperform many of the available open source and closed chat models on common industry benchmarks. + +Model Developer: Meta + +Model Architecture: Llama 3.2 is an auto-regressive language model that uses an optimized transformer architecture. The tuned versions use supervised fine-tuning (SFT) and reinforcement learning with human feedback (RLHF) to align with human preferences for helpfulness and safety. + +| | Training Data | Params | Input modalities | Output modalities | Context Length | GQA | Shared Embeddings | Token count | Knowledge cutoff | +| :---- | :---- | :---- | :---- | :---- | :---- | :---- | :---- | :---- | :---- | +| Llama 3.2 (text only) | A new mix of publicly available online data. | 1B (1.23B) | Multilingual Text | Multilingual Text and code | 128k | Yes | Yes | Up to 9T tokens | December 2023 | +| | | 3B (3.21B) | Multilingual Text | Multilingual Text and code | | | | | | +| Llama 3.2 Quantized (text only) | A new mix of publicly available online data. | 1B (1.23B) | Multilingual Text | Multilingual Text and code | 8k | Yes | Yes | Up to 9T tokens | December 2023 | +| | | 3B (3.21B) | Multilingual Text | Multilingual Text and code | | | | | | + +Supported Languages: English, German, French, Italian, Portuguese, Hindi, Spanish, and Thai are officially supported. Llama 3.2 has been trained on a broader collection of languages than these 8 supported languages. Developers may fine-tune Llama 3.2 models for languages beyond these supported languages, provided they comply with the Llama 3.2 Community License and the Acceptable Use Policy. Developers are always expected to ensure that their deployments, including those that involve additional languages, are completed safely and responsibly. + +Llama 3.2 Model Family: Token counts refer to pretraining data only. All model versions use Grouped-Query Attention (GQA) for improved inference scalability. + +Model Release Date: Oct 24, 2024 + +Status: This is a static model trained on an offline dataset. Future versions may be released that improve model capabilities and safety. + +License: Use of Llama 3.2 is governed by the [Llama 3.2 Community License](https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/LICENSE) (a custom, commercial license agreement). + +Feedback: Instructions on how to provide feedback or comments on the model can be found in the Llama Models [README](https://github.com/meta-llama/llama-models/blob/main/README.md). For more technical information about generation parameters and recipes for how to use Llama 3.2 in applications, please go [here](https://github.com/meta-llama/llama-recipes). + +## Intended Use + +Intended Use Cases: Llama 3.2 is intended for commercial and research use in multiple languages. Instruction tuned text only models are intended for assistant-like chat and agentic applications like knowledge retrieval and summarization, mobile AI powered writing assistants and query and prompt rewriting. Pretrained models can be adapted for a variety of additional natural language generation tasks. Similarly, quantized models can be adapted for a variety of on-device use-cases with limited compute resources. + +Out of Scope: Use in any manner that violates applicable laws or regulations (including trade compliance laws). Use in any other way that is prohibited by the Acceptable Use Policy and Llama 3.2 Community License. Use in languages beyond those explicitly referenced as supported in this model card. + +## Hardware and Software + +Training Factors: We used custom training libraries, Meta's custom built GPU cluster, and production infrastructure for pretraining. Fine-tuning, quantization, annotation, and evaluation were also performed on production infrastructure. + +Training Energy Use: Training utilized a cumulative of 916k GPU hours of computation on H100-80GB (TDP of 700W) type hardware, per the table below. Training time is the total GPU time required for training each model and power consumption is the peak power capacity per GPU device used, adjusted for power usage efficiency. + +Training Greenhouse Gas Emissions: Estimated total location-based greenhouse gas emissions were 240 tons CO2eq for training. Since 2020, Meta has maintained net zero greenhouse gas emissions in its global operations and matched 100% of its electricity use with renewable energy; therefore, the total market-based greenhouse gas emissions for training were 0 tons CO2eq. + +| | Training Time (GPU hours) | Logit Generation Time (GPU Hours) | Training Power Consumption (W) | Training Location-Based Greenhouse Gas Emissions (tons CO2eq) | Training Market-Based Greenhouse Gas Emissions (tons CO2eq) | +| :---- | :---: | ----- | :---: | :---: | :---: | +| Llama 3.2 1B | 370k | \- | 700 | 107 | 0 | +| Llama 3.2 3B | 460k | \- | 700 | 133 | 0 | +| Llama 3.2 1B SpinQuant | 1.7 | 0 | 700 | *Negligible*\*\* | 0 | +| Llama 3.2 3B SpinQuant | 2.4 | 0 | 700 | *Negligible*\*\* | 0 | +| Llama 3.2 1B QLoRA | 1.3k | 0 | 700 | 0.381 | 0 | +| Llama 3.2 3B QLoRA | 1.6k | 0 | 700 | 0.461 | 0 | +| Total | 833k | 86k | | 240 | 0 | + +\*\* The location-based CO2e emissions of Llama 3.2 1B SpinQuant and Llama 3.2 3B SpinQuant are less than 0.001 metric tonnes each. This is due to the minimal training GPU hours that are required. + +The methodology used to determine training energy use and greenhouse gas emissions can be found [here](https://arxiv.org/pdf/2204.05149). Since Meta is openly releasing these models, the training energy use and greenhouse gas emissions will not be incurred by others. + +## Training Data + +Overview: Llama 3.2 was pretrained on up to 9 trillion tokens of data from publicly available sources. For the 1B and 3B Llama 3.2 models, we incorporated logits from the Llama 3.1 8B and 70B models into the pretraining stage of the model development, where outputs (logits) from these larger models were used as token-level targets. Knowledge distillation was used after pruning to recover performance. In post-training we used a similar recipe as Llama 3.1 and produced final chat models by doing several rounds of alignment on top of the pre-trained model. Each round involved Supervised Fine-Tuning (SFT), Rejection Sampling (RS), and Direct Preference Optimization (DPO). + +Data Freshness: The pretraining data has a cutoff of December 2023\. + +## Quantization + +### Quantization Scheme + +We designed the current quantization scheme with the [PyTorch’s ExecuTorch](https://github.com/pytorch/executorch) inference framework and Arm CPU backend in mind, taking into account metrics including model quality, prefill/decoding speed, and memory footprint. Our quantization scheme involves three parts: +- All linear layers in all transformer blocks are quantized to a 4-bit groupwise scheme (with a group size of 32) for weights and 8-bit per-token dynamic quantization for activations. +- The classification layer is quantized to 8-bit per-channel for weight and 8-bit per token dynamic quantization for activation. +- Similar to classification layer, an 8-bit per channel quantization is used for embedding layer. + + +### Quantization-Aware Training and LoRA + +The quantization-aware training (QAT) with low-rank adaptation (LoRA) models went through only post-training stages, using the same data as the full precision models. To initialize QAT, we utilize BF16 Llama 3.2 model checkpoints obtained after supervised fine-tuning (SFT) and perform an additional full round of SFT training with QAT. We used the [torchao](https://github.com/pytorch/ao/blob/main/torchao/quantization/qat/README.md) API for this. We then freeze the backbone of the QAT model and perform another round of SFT with LoRA adaptors applied to all layers within the transformer block. Meanwhile, the LoRA adaptors' weights and activations are maintained in BF16. Because our approach is similar to QLoRA of Dettmers et al., (2023) (i.e., quantization followed by LoRA adapters), we refer this method as QLoRA. Finally, we fine-tune the resulting model (both backbone and LoRA adaptors) using direct preference optimization (DPO). + +### SpinQuant + +[SpinQuant](https://arxiv.org/abs/2405.16406) was applied, together with generative post-training quantization (GPTQ). For the SpinQuant rotation matrix fine-tuning, we optimized for 100 iterations, using 800 samples with sequence-length 2048 from the WikiText 2 dataset. For GPTQ, we used 128 samples from the same dataset with the same sequence-length. + +## Benchmarks \- English Text + +In this section, we report the results for Llama 3.2 models on standard automatic benchmarks. For all these evaluations, we used our internal evaluations library. + +### Base Pretrained Models + +| Category | Benchmark | \# Shots | Metric | Llama 3.2 1B | Llama 3.2 3B | Llama 3.1 8B | +| ----- | ----- | :---: | :---: | :---: | :---: | :---: | +| General | MMLU | 5 | macro\_avg/acc\_char | 32.2 | 58 | 66.7 | +| | AGIEval English | 3-5 | average/acc\_char | 23.3 | 39.2 | 47.8 | +| | ARC-Challenge | 25 | acc\_char | 32.8 | 69.1 | 79.7 | +| Reading comprehension | SQuAD | 1 | em | 49.2 | 67.7 | 77 | +| | QuAC (F1) | 1 | f1 | 37.9 | 42.9 | 44.9 | +| | DROP (F1) | 3 | f1 | 28.0 | 45.2 | 59.5 | +| Long Context | Needle in Haystack | 0 | em | 96.8 | 1 | 1 | + +### Instruction Tuned Models + +| Capability | | Benchmark | \# Shots | Metric | Llama 3.2 1B bf16 | Llama 3.2 1B Vanilla PTQ\*\* | Llama 3.2 1B Spin Quant | Llama 3.2 1B QLoRA | Llama 3.2 3B bf16 | Llama 3.2 3B Vanilla PTQ\*\* | Llama 3.2 3B Spin Quant | Llama 3.2 3B QLoRA | Llama 3.1 8B | +| :---: | ----- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| General | | MMLU | 5 | macro\_avg/acc | 49.3 | 43.3 | 47.3 | 49.0 | 63.4 | 60.5 | 62 | 62.4 | 69.4 | +| Re-writing | | Open-rewrite eval | 0 | micro\_avg/rougeL | 41.6 | 39.2 | 40.9 | 41.2 | 40.1 | 40.3 | 40.8 | 40.7 | 40.9 | +| Summarization | | TLDR9+ (test) | 1 | rougeL | 16.8 | 14.9 | 16.7 | 16.8 | 19.0 | 19.1 | 19.2 | 19.1 | 17.2 | +| Instruction following | | IFEval | 0 | Avg(Prompt/Instruction acc Loose/Strict) | 59.5 | 51.5 | 58.4 | 55.6 | 77.4 | 73.9 | 73.5 | 75.9 | 80.4 | +| Math | | GSM8K (CoT) | 8 | em\_maj1@1 | 44.4 | 33.1 | 40.6 | 46.5 | 77.7 | 72.9 | 75.7 | 77.9 | 84.5 | +| | | MATH (CoT) | 0 | final\_em | 30.6 | 20.5 | 25.3 | 31.0 | 48.0 | 44.2 | 45.3 | 49.2 | 51.9 | +| Reasoning | | ARC-C | 0 | acc | 59.4 | 54.3 | 57 | 60.7 | 78.6 | 75.6 | 77.6 | 77.6 | 83.4 | +| | | GPQA | 0 | acc | 27.2 | 25.9 | 26.3 | 25.9 | 32.8 | 32.8 | 31.7 | 33.9 | 32.8 | +| | | Hellaswag | 0 | acc | 41.2 | 38.1 | 41.3 | 41.5 | 69.8 | 66.3 | 68 | 66.3 | 78.7 | +| Tool Use | | BFCL V2 | 0 | acc | 25.7 | 14.3 | 15.9 | 23.7 | 67.0 | 53.4 | 60.1 | 63.5 | 67.1 | +| | | Nexus | 0 | macro\_avg/acc | 13.5 | 5.2 | 9.6 | 12.5 | 34.3 | 32.4 | 31.5 | 30.1 | 38.5 | +| Long Context | | InfiniteBench/En.QA | 0 | longbook\_qa/f1 | 20.3 | N/A | N/A | N/A | 19.8 | N/A | N/A | N/A | 27.3 | +| | | InfiniteBench/En.MC | 0 | longbook\_choice/acc | 38.0 | N/A | N/A | N/A | 63.3 | N/A | N/A | N/A | 72.2 | +| | | NIH/Multi-needle | 0 | recall | 75.0 | N/A | N/A | N/A | 84.7 | N/A | N/A | N/A | 98.8 | +| Multilingual | | MGSM (CoT) | 0 | em | 24.5 | 13.7 | 18.2 | 24.4 | 58.2 | 48.9 | 54.3 | 56.8 | 68.9 | + +\*\*for comparison purposes only. Model not released. + +### Multilingual Benchmarks + +| Category | Benchmark | Language | Llama 3.2 1B | Llama 3.2 1B Vanilla PTQ\*\* | Llama 3.2 1B Spin Quant | Llama 3.2 1B QLoRA | Llama 3.2 3B | Llama 3.2 3B Vanilla PTQ\*\* | Llama 3.2 3B Spin Quant | Llama 3.2 3B QLoRA | Llama 3.1 8B | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| General | MMLU (5-shot, macro_avg/acc) | Portuguese | 39.8 | 34.9 | 38.9 | 40.2 | 54.5 | 50.9 | 53.3 | 53.4 | 62.1 | +| | | Spanish | 41.5 | 36.0 | 39.8 | 41.8 | 55.1 | 51.9 | 53.6 | 53.6 | 62.5 | +| | | Italian | 39.8 | 34.9 | 38.1 | 40.6 | 53.8 | 49.9 | 52.1 | 51.7 | 61.6 | +| | | German | 39.2 | 34.9 | 37.5 | 39.6 | 53.3 | 50.0 | 52.2 | 51.3 | 60.6 | +| | | French | 40.5 | 34.8 | 39.2 | 40.8 | 54.6 | 51.2 | 53.3 | 53.3 | 62.3 | +| | | Hindi | 33.5 | 30.0 | 32.1 | 34.0 | 43.3 | 40.4 | 42.0 | 42.1 | 50.9 | +| | | Thai | 34.7 | 31.2 | 32.4 | 34.9 | 44.5 | 41.3 | 44.0 | 42.2 | 50.3 | + +\*\*for comparison purposes only. Model not released. + +## Inference time + +In the below table, we compare the performance metrics of different quantization methods (SpinQuant and QAT \+ LoRA) with the BF16 baseline. The evaluation was done using the [ExecuTorch](https://github.com/pytorch/executorch) framework as the inference engine, with the ARM CPU as a backend using Android OnePlus 12 device. + +| Category | Decode (tokens/sec) | Time-to-first-token (sec) | Prefill (tokens/sec) | Model size (PTE file size in MB) | Memory size (RSS in MB) | +| :---- | ----- | ----- | ----- | ----- | ----- | +| 1B BF16 (baseline) | 19.2 | 1.0 | 60.3 | 2358 | 3,185 | +| 1B SpinQuant | 50.2 (2.6x) | 0.3 (-76.9%) | 260.5 (4.3x) | 1083 (-54.1%) | 1,921 (-39.7%) | +| 1B QLoRA | 45.8 (2.4x) | 0.3 (-76.0%) | 252.0 (4.2x) | 1127 (-52.2%) | 2,255 (-29.2%) | +| 3B BF16 (baseline) | 7.6 | 3.0 | 21.2 | 6129 | 7,419 | +| 3B SpinQuant | 19.7 (2.6x) | 0.7 (-76.4%) | 89.7 (4.2x) | 2435 (-60.3%) | 3,726 (-49.8%) | +| 3B QLoRA | 18.5 (2.4x) | 0.7 (-76.1%) | 88.8 (4.2x) | 2529 (-58.7%) | 4,060 (-45.3%) | + +(\*) The performance measurement is done using an adb binary-based approach. +(\*\*) It is measured on an Android OnePlus 12 device. +(\*\*\*) Time-to-first-token (TTFT) is measured with prompt length=64 + +*Footnote:* + +- *Decode (tokens/second) is for how quickly it keeps generating. Higher is better.* +- *Time-to-first-token (TTFT for shorthand) is for how fast it generates the first token for a given prompt. Lower is better.* +- *Prefill is the inverse of TTFT (aka 1/TTFT) in tokens/second. Higher is better* +- *Model size \- how big is the model, measured by, PTE file, a binary file format for ExecuTorch* +- *RSS size \- Memory usage in resident set size (RSS)* + +## Responsibility & Safety + +As part of our Responsible release approach, we followed a three-pronged strategy to managing trust & safety risks: + +1. Enable developers to deploy helpful, safe and flexible experiences for their target audience and for the use cases supported by Llama +2. Protect developers against adversarial users aiming to exploit Llama capabilities to potentially cause harm +3. Provide protections for the community to help prevent the misuse of our models + +### Responsible Deployment + +Approach: Llama is a foundational technology designed to be used in a variety of use cases. Examples on how Meta’s Llama models have been responsibly deployed can be found in our [Community Stories webpage](https://llama.meta.com/community-stories/). Our approach is to build the most helpful models, enabling the world to benefit from the technology power, by aligning our model safety for generic use cases and addressing a standard set of harms. Developers are then in the driver’s seat to tailor safety for their use cases, defining their own policies and deploying the models with the necessary safeguards in their Llama systems. Llama 3.2 was developed following the best practices outlined in our [Responsible Use Guide](https://llama.meta.com/responsible-use-guide/). + +#### Llama 3.2 Instruct + +Objective: Our main objectives for conducting safety fine-tuning are to provide the research community with a valuable resource for studying the robustness of safety fine-tuning, as well as to offer developers a readily available, safe, and powerful model for various applications to reduce the developer workload to deploy safe AI systems. We implemented the same set of safety mitigations as in Llama 3, and you can learn more about these in the Llama 3 [paper](https://ai.meta.com/research/publications/the-llama-3-herd-of-models/). + +Fine-Tuning Data: We employ a multi-faceted approach to data collection, combining human-generated data from our vendors with synthetic data to mitigate potential safety risks. We’ve developed many large language model (LLM)-based classifiers that enable us to thoughtfully select high-quality prompts and responses, enhancing data quality control. + +Refusals and Tone: Building on the work we started with Llama 3, we put a great emphasis on model refusals to benign prompts as well as refusal tone. We included both borderline and adversarial prompts in our safety data strategy, and modified our safety data responses to follow tone guidelines. + +#### Llama 3.2 Systems + +Safety as a System: Large language models, including Llama 3.2, are not designed to be deployed in isolation but instead should be deployed as part of an overall AI system with additional safety guardrails as required. Developers are expected to deploy system safeguards when building agentic systems. Safeguards are key to achieve the right helpfulness-safety alignment as well as mitigating safety and security risks inherent to the system and any integration of the model or system with external tools. As part of our responsible release approach, we provide the community with [safeguards](https://llama.meta.com/trust-and-safety/) that developers should deploy with Llama models or other LLMs, including Llama Guard, Prompt Guard and Code Shield. All our [reference implementations](https://github.com/meta-llama/llama-agentic-system) demos contain these safeguards by default so developers can benefit from system-level safety out-of-the-box. + +### New Capabilities and Use Cases + +Technological Advancement: Llama releases usually introduce new capabilities that require specific considerations in addition to the best practices that generally apply across all Generative AI use cases. For prior release capabilities also supported by Llama 3.2, see [Llama 3.1 Model Card](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md), as the same considerations apply here as well. + +Constrained Environments: Llama 3.2 1B and 3B models are expected to be deployed in highly constrained environments, such as mobile devices. LLM Systems using smaller models will have a different alignment profile and safety/helpfulness tradeoff than more complex, larger systems. Developers should ensure the safety of their system meets the requirements of their use case. We recommend using lighter system safeguards for such use cases, like Llama Guard 3-1B or its mobile-optimized version. + +### Evaluations + +Scaled Evaluations: We built dedicated, adversarial evaluation datasets and evaluated systems composed of Llama models and Purple Llama safeguards to filter input prompt and output response. It is important to evaluate applications in context, and we recommend building dedicated evaluation dataset for your use case. + +Red Teaming: We conducted recurring red teaming exercises with the goal of discovering risks via adversarial prompting and we used the learnings to improve our benchmarks and safety tuning datasets. We partnered early with subject-matter experts in critical risk areas to understand the nature of these real-world harms and how such models may lead to unintended harm for society. Based on these conversations, we derived a set of adversarial goals for the red team to attempt to achieve, such as extracting harmful information or reprogramming the model to act in a potentially harmful capacity. The red team consisted of experts in cybersecurity, adversarial machine learning, responsible AI, and integrity in addition to multilingual content specialists with background in integrity issues in specific geographic markets. + +### Critical Risks + +In addition to our safety work above, we took extra care on measuring and/or mitigating the following critical risk areas: + +1\. CBRNE (Chemical, Biological, Radiological, Nuclear, and Explosive Weapons): Llama 3.2 1B and 3B models are smaller and less capable derivatives of Llama 3.1. For Llama 3.1 70B and 405B, to assess risks related to proliferation of chemical and biological weapons, we performed uplift testing designed to assess whether use of Llama 3.1 models could meaningfully increase the capabilities of malicious actors to plan or carry out attacks using these types of weapons and have determined that such testing also applies to the smaller 1B and 3B models. + +2\. Child Safety: Child Safety risk assessments were conducted using a team of experts, to assess the model’s capability to produce outputs that could result in Child Safety risks and inform on any necessary and appropriate risk mitigations via fine tuning. We leveraged those expert red teaming sessions to expand the coverage of our evaluation benchmarks through Llama 3 model development. For Llama 3, we conducted new in-depth sessions using objective based methodologies to assess the model risks along multiple attack vectors including the additional languages Llama 3 is trained on. We also partnered with content specialists to perform red teaming exercises assessing potentially violating content while taking account of market specific nuances or experiences. + +3\. Cyber Attacks: For Llama 3.1 405B, our cyber attack uplift study investigated whether LLMs can enhance human capabilities in hacking tasks, both in terms of skill level and speed. +Our attack automation study focused on evaluating the capabilities of LLMs when used as autonomous agents in cyber offensive operations, specifically in the context of ransomware attacks. This evaluation was distinct from previous studies that considered LLMs as interactive assistants. The primary objective was to assess whether these models could effectively function as independent agents in executing complex cyber-attacks without human intervention. Because Llama 3.2’s 1B and 3B models are smaller and less capable models than Llama 3.1 405B, we broadly believe that the testing conducted for the 405B model also applies to Llama 3.2 models. + +### Community + +Industry Partnerships: Generative AI safety requires expertise and tooling, and we believe in the strength of the open community to accelerate its progress. We are active members of open consortiums, including the AI Alliance, Partnership on AI and MLCommons, actively contributing to safety standardization and transparency. We encourage the community to adopt taxonomies like the MLCommons Proof of Concept evaluation to facilitate collaboration and transparency on safety and content evaluations. Our Purple Llama tools are open sourced for the community to use and widely distributed across ecosystem partners including cloud service providers. We encourage community contributions to our [Github repository](https://github.com/meta-llama/PurpleLlama). + +Grants: We also set up the [Llama Impact Grants](https://llama.meta.com/llama-impact-grants/) program to identify and support the most compelling applications of Meta’s Llama model for societal benefit across three categories: education, climate and open innovation. The 20 finalists from the hundreds of applications can be found [here](https://llama.meta.com/llama-impact-grants/#finalists). + +Reporting: Finally, we put in place a set of resources including an [output reporting mechanism](https://developers.facebook.com/llama_output_feedback) and [bug bounty program](https://www.facebook.com/whitehat) to continuously improve the Llama technology with the help of the community. + +## Ethical Considerations and Limitations + +Values: The core values of Llama 3.2 are openness, inclusivity and helpfulness. It is meant to serve everyone, and to work for a wide range of use cases. It is thus designed to be accessible to people across many different backgrounds, experiences and perspectives. Llama 3.2 addresses users and their needs as they are, without insertion unnecessary judgment or normativity, while reflecting the understanding that even content that may appear problematic in some cases can serve valuable purposes in others. It respects the dignity and autonomy of all users, especially in terms of the values of free thought and expression that power innovation and progress. + +Testing: Llama 3.2 is a new technology, and like any new technology, there are risks associated with its use. Testing conducted to date has not covered, nor could it cover, all scenarios. For these reasons, as with all LLMs, Llama 3.2’s potential outputs cannot be predicted in advance, and the model may in some instances produce inaccurate, biased or other objectionable responses to user prompts. Therefore, before deploying any applications of Llama 3.2 models, developers should perform safety testing and tuning tailored to their specific applications of the model. Please refer to available resources including our [Responsible Use Guide](https://llama.meta.com/responsible-use-guide), [Trust and Safety](https://llama.meta.com/trust-and-safety/) solutions, and other [resources](https://llama.meta.com/docs/get-started/) to learn more about responsible development. diff --git a/examples/DocQA/example_data/llama_3.2_vision.md b/examples/DocQA/example_data/llama_3.2_vision.md new file mode 100644 index 000000000..e64eee2d1 --- /dev/null +++ b/examples/DocQA/example_data/llama_3.2_vision.md @@ -0,0 +1,160 @@ +## Model Information + +The Llama 3.2-Vision collection of multimodal large language models (LLMs) is a collection of pretrained and instruction-tuned image reasoning generative models in 11B and 90B sizes (text \+ images in / text out). The Llama 3.2-Vision instruction-tuned models are optimized for visual recognition, image reasoning, captioning, and answering general questions about an image. The models outperform many of the available open source and closed multimodal models on common industry benchmarks. + +Model Developer: Meta + +Model Architecture: Llama 3.2-Vision is built on top of the Llama 3.1 text-only model, which is an auto-regressive language model that uses an optimized transformer architecture. The tuned versions use supervised fine-tuning (SFT) and reinforcement learning with human feedback (RLHF) to align with human preferences for helpfulness and safety. To support image recognition tasks, the Llama 3.2-Vision model uses a separately trained vision adapter that integrates with the pre-trained Llama 3.1 language model. The adapter consists of a series of cross-attention layers that feed image encoder representations into the core LLM. + +| | Training Data | Params | Input modalities | Output modalities | Context length | GQA | Data volume | Knowledge cutoff | +| :---- | :---- | :---- | :---- | :---- | :---- | :---- | :---- | :---- | +| Llama 3.2-Vision | (Image, text) pairs | 11B (10.6) | Text \+ Image | Text | 128k | Yes | 6B (image, text) pairs | December 2023 | +| Llama 3.2-Vision | (Image, text) pairs | 90B (88.8) | Text \+ Image | Text | 128k | Yes | 6B (image, text) pairs | December 2023 | + +Supported Languages: For text only tasks, English, German, French, Italian, Portuguese, Hindi, Spanish, and Thai are officially supported. Llama 3.2 has been trained on a broader collection of languages than these 8 supported languages. Note for image+text applications, English is the only language supported. + +Developers may fine-tune Llama 3.2 models for languages beyond these supported languages, provided they comply with the Llama 3.2 Community License and the Acceptable Use Policy. Developers are always expected to ensure that their deployments, including those that involve additional languages, are completed safely and responsibly. + +Llama 3.2 Model Family: Token counts refer to pretraining data only. All model versions use Grouped-Query Attention (GQA) for improved inference scalability. + +Model Release Date: Sept 25, 2024 + +Status: This is a static model trained on an offline dataset. Future versions may be released that improve model capabilities and safety. + +License: Use of Llama 3.2 is governed by the [Llama 3.2 Community License](https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/LICENSE) (a custom, commercial license agreement). + +Feedback: Instructions on how to provide feedback or comments on the model can be found in the Llama Models [README](https://github.com/meta-llama/llama-models/blob/main/README.md). For more technical information about generation parameters and recipes for how to use Llama 3.2-Vision in applications, please go [here](https://github.com/meta-llama/llama-recipes). + +## Intended Use + +Intended Use Cases: Llama 3.2-Vision is intended for commercial and research use. Instruction tuned models are intended for visual recognition, image reasoning, captioning, and assistant-like chat with images, whereas pretrained models can be adapted for a variety of image reasoning tasks. Additionally, because of Llama 3.2-Vision’s ability to take images and text as inputs, additional use cases could include: + +1. Visual Question Answering (VQA) and Visual Reasoning: Imagine a machine that looks at a picture and understands your questions about it. +2. Document Visual Question Answering (DocVQA): Imagine a computer understanding both the text and layout of a document, like a map or contract, and then answering questions about it directly from the image. +3. Image Captioning: Image captioning bridges the gap between vision and language, extracting details, understanding the scene, and then crafting a sentence or two that tells the story. +4. Image-Text Retrieval: Image-text retrieval is like a matchmaker for images and their descriptions. Similar to a search engine but one that understands both pictures and words. +5. Visual Grounding: Visual grounding is like connecting the dots between what we see and say. It’s about understanding how language references specific parts of an image, allowing AI models to pinpoint objects or regions based on natural language descriptions. + + +The Llama 3.2 model collection also supports the ability to leverage the outputs of its models to improve other models including synthetic data generation and distillation. The Llama 3.2 Community License allows for these use cases. + +Out of Scope: Use in any manner that violates applicable laws or regulations (including trade compliance laws). Use in any other way that is prohibited by the Acceptable Use Policy and Llama 3.2 Community License. Use in languages beyond those explicitly referenced as supported in this model card. + +## Hardware and Software + +Training Factors: We used custom training libraries, Meta's custom built GPU cluster, and production infrastructure for pretraining. Fine-tuning, annotation, and evaluation were also performed on production infrastructure. + +Training Energy Use: Training utilized a cumulative of 2.02M GPU hours of computation on H100-80GB (TDP of 700W) type hardware, per the table below. Training time is the total GPU time required for training each model and power consumption is the peak power capacity per GPU device used, adjusted for power usage efficiency. + +## + +Training Greenhouse Gas Emissions: Estimated total location-based greenhouse gas emissions were 584 tons CO2eq for training. Since 2020, Meta has maintained net zero greenhouse gas emissions in its global operations and matched 100% of its electricity use with renewable energy, therefore the total market-based greenhouse gas emissions for training were 0 tons CO2eq. + +| | Training Time (GPU hours) | Training Power Consumption (W) | Training Location-Based Greenhouse Gas Emissions (tons CO2eq) | Training Market-Based Greenhouse Gas Emissions (tons CO2eq) | +| :---- | :---: | :---: | :---: | :---: | +| Llama 3.2-vision 11B | Stage 1 pretraining: 147K H100 hours Stage 2 annealing: 98K H100 hours SFT: 896 H100 hours RLHF: 224 H100 hours | 700 | 71 | 0 | +| Llama 3.2-vision 90B | Stage 1 pretraining: 885K H100 hours Stage 2 annealing: 885K H100 hours SFT: 3072 H100 hours RLHF: 2048 H100 hours | 700 | 513 | 0 | +| Total | 2.02M | | 584 | 0 | + +The methodology used to determine training energy use and greenhouse gas emissions can be found [here](https://arxiv.org/pdf/2204.05149). Since Meta is openly releasing these models, the training energy use and greenhouse gas emissions will not be incurred by others. + +## Training Data + +Overview: Llama 3.2-Vision was pretrained on 6B image and text pairs. The instruction tuning data includes publicly available vision instruction datasets, as well as over 3M synthetically generated examples. + +Data Freshness: The pretraining data has a cutoff of December 2023\. + +## Benchmarks \- Image Reasoning + +In this section, we report the results for Llama 3.2-Vision models on standard automatic benchmarks. For all these evaluations, we used our internal evaluations library. + +### Base Pretrained Models + +| Category | Benchmark | \# Shots | Metric | Llama 3.2 11B | Llama 3.2 90B | +| ----- | ----- | ----- | ----- | ----- | ----- | +| Image Understanding | VQAv2 (val) | 0 | Accuracy | 66.8 | 73.6 | +| | Text VQA (val) | 0 | Relaxed accuracy | 73.1 | 73.5 | +| | DocVQA (val, unseen) | 0 | ANLS | 62.3 | 70.7 | +| Visual Reasoning | MMMU (val, 0-shot) | 0 | Micro average accuracy | 41.7 | 49.3 | +| | ChartQA (test) | 0 | Accuracy | 39.4 | 54.2 | +| | InfographicsQA (val, unseen) | 0 | ANLS | 43.2 | 56.8 | +| | AI2 Diagram (test) | 0 | Accuracy | 62.4 | 75.3 | + +### Instruction Tuned Models + +| Modality | Capability | Benchmark | \# Shots | Metric | Llama 3.2 11B | Llama 3.2 90B | +| ----- | :---: | ----- | :---: | :---: | ----- | ----- | +| Image | College-level Problems and Mathematical Reasoning | MMMU (val, CoT) | 0 | Micro average accuracy | 50.7 | 60.3 | +| | | MMMU-Pro, Standard (10 opts, test) | 0 | Accuracy | 33.0 | 45.2 | +| | | MMMU-Pro, Vision (test) | 0 | Accuracy | 23.7 | 33.8 | +| | | MathVista (testmini) | 0 | Accuracy | 51.5 | 57.3 | +| | Charts and Diagram Understanding | ChartQA (test, CoT) | 0 | Relaxed accuracy | 83.4 | 85.5 | +| | | AI2 Diagram (test) | 0 | Accuracy | 91.1 | 92.3 | +| | | DocVQA (test) | 0 | ANLS | 88.4 | 90.1 | +| | General Visual Question Answering | VQAv2 (test) | 0 | Accuracy | 75.2 | 78.1 | +| | | | | | | | +| Text | General | MMLU (CoT) | 0 | Macro\_avg/acc | 73.0 | 86.0 | +| | Math | MATH (CoT) | 0 | Final\_em | 51.9 | 68.0 | +| | Reasoning | GPQA | 0 | Accuracy | 32.8 | 46.7 | +| | Multilingual | MGSM (CoT) | 0 | em | 68.9 | 86.9 | + +## Responsibility & Safety + +As part of our Responsible release approach, we followed a three-pronged strategy to managing trust & safety risks: + +1. Enable developers to deploy helpful, safe and flexible experiences for their target audience and for the use cases supported by Llama. +2. Protect developers against adversarial users aiming to exploit Llama capabilities to potentially cause harm. +3. Provide protections for the community to help prevent the misuse of our models. + +### Responsible Deployment + +Approach: Llama is a foundational technology designed to be used in a variety of use cases, examples on how Meta’s Llama models have been responsibly deployed can be found in our [Community Stories webpage](https://llama.meta.com/community-stories/). Our approach is to build the most helpful models enabling the world to benefit from the technology power, by aligning our model safety for the generic use cases addressing a standard set of harms. Developers are then in the driver seat to tailor safety for their use case, defining their own policy and deploying the models with the necessary safeguards in their Llama systems. Llama 3.2 was developed following the best practices outlined in our Responsible Use Guide, you can refer to the [Responsible Use Guide](https://llama.meta.com/responsible-use-guide/) to learn more. + +#### Llama 3.2 Instruct + +Objective: Our main objectives for conducting safety fine-tuning are to provide the research community with a valuable resource for studying the robustness of safety fine-tuning, as well as to offer developers a readily available, safe, and powerful model for various applications to reduce the developer workload to deploy safe AI systems. We implemented the same set of safety mitigations as in Llama 3, and you can learn more about these in the Llama 3 [paper](https://ai.meta.com/research/publications/the-llama-3-herd-of-models/). + +Fine-Tuning Data: We employ a multi-faceted approach to data collection, combining human-generated data from our vendors with synthetic data to mitigate potential safety risks. We’ve developed many large language model (LLM)-based classifiers that enable us to thoughtfully select high-quality prompts and responses, enhancing data quality control. + +Refusals and Tone: Building on the work we started with Llama 3, we put a great emphasis on model refusals to benign prompts as well as refusal tone. We included both borderline and adversarial prompts in our safety data strategy, and modified our safety data responses to follow tone guidelines. + +#### Llama 3.2 Systems + +Safety as a System: Large language models, including Llama 3.2, are not designed to be deployed in isolation but instead should be deployed as part of an overall AI system with additional safety guardrails as required. Developers are expected to deploy system safeguards when building agentic systems. Safeguards are key to achieve the right helpfulness-safety alignment as well as mitigating safety and security risks inherent to the system and any integration of the model or system with external tools. As part of our responsible release approach, we provide the community with [safeguards](https://llama.meta.com/trust-and-safety/) that developers should deploy with Llama models or other LLMs, including Llama Guard, Prompt Guard and Code Shield. All our [reference implementations](https://github.com/meta-llama/llama-agentic-system) demos contain these safeguards by default so developers can benefit from system-level safety out-of-the-box. + +### New Capabilities and Use Cases + +Technological Advancement: Llama releases usually introduce new capabilities that require specific considerations in addition to the best practices that generally apply across all Generative AI use cases. For prior release capabilities also supported by Llama 3.2, see [Llama 3.1 Model Card](https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/MODEL_CARD.md), as the same considerations apply here as well. + +Image Reasoning: Llama 3.2-Vision models come with multimodal (text and image) input capabilities enabling image reasoning applications. As part of our responsible release process, we took dedicated measures including evaluations and mitigations to address the risk of the models uniquely identifying individuals in images. As with other LLM risks, models may not always be robust to adversarial prompts, and developers should evaluate identification and other applicable risks in the context of their applications as well as consider deploying Llama Guard 3-11B-Vision as part of their system or other mitigations as appropriate to detect and mitigate such risks. + +### Evaluations + +Scaled Evaluations: We built dedicated, adversarial evaluation datasets and evaluated systems composed of Llama models and Purple Llama safeguards to filter input prompt and output response. It is important to evaluate applications in context, and we recommend building dedicated evaluation dataset for your use case. + +Red teaming: We conducted recurring red teaming exercises with the goal of discovering risks via adversarial prompting and we used the learnings to improve our benchmarks and safety tuning datasets. We partnered early with subject-matter experts in critical risk areas to understand the nature of these real-world harms and how such models may lead to unintended harm for society. Based on these conversations, we derived a set of adversarial goals for the red team to attempt to achieve, such as extracting harmful information or reprogramming the model to act in a potentially harmful capacity. The red team consisted of experts in cybersecurity, adversarial machine learning, responsible AI, and integrity in addition to multilingual content specialists with background in integrity issues in specific geographic markets. + +### Critical Risks + +In addition to our safety work above, we took extra care on measuring and/or mitigating the following critical risk areas: + +1\. CBRNE (Chemical, Biological, Radiological, Nuclear, and Explosive Weapons): For Llama 3.1, to assess risks related to proliferation of chemical and biological weapons, we performed uplift testing designed to assess whether use of Llama 3.1 models could meaningfully increase the capabilities of malicious actors to plan or carry out attacks using these types of weapons. For Llama 3.2-Vision models, we conducted additional targeted evaluations and found that it was unlikely Llama 3.2 presented an increase in scientific capabilities due to its added image understanding capability as compared to Llama 3.1. + +2\. Child Safety: Child Safety risk assessments were conducted using a team of experts, to assess the model’s capability to produce outputs that could result in Child Safety risks and inform on any necessary and appropriate risk mitigations via fine tuning. We leveraged those expert red teaming sessions to expand the coverage of our evaluation benchmarks through Llama 3 model development. For Llama 3, we conducted new in-depth sessions using objective based methodologies to assess the model risks along multiple attack vectors including the additional languages Llama 3 is trained on. We also partnered with content specialists to perform red teaming exercises assessing potentially violating content while taking account of market specific nuances or experiences. + +3\. Cyber Attacks: For Llama 3.1 405B, our cyber attack uplift study investigated whether LLMs can enhance human capabilities in hacking tasks, both in terms of skill level and speed. +Our attack automation study focused on evaluating the capabilities of LLMs when used as autonomous agents in cyber offensive operations, specifically in the context of ransomware attacks. This evaluation was distinct from previous studies that considered LLMs as interactive assistants. The primary objective was to assess whether these models could effectively function as independent agents in executing complex cyber-attacks without human intervention. Because Llama 3.2’s vision capabilities are not generally germane to cyber uplift, we believe that the testing conducted for Llama 3.1 also applies to Llama 3.2. + +### Community + +Industry Partnerships: Generative AI safety requires expertise and tooling, and we believe in the strength of the open community to accelerate its progress. We are active members of open consortiums, including the AI Alliance, Partnership on AI and MLCommons, actively contributing to safety standardization and transparency. We encourage the community to adopt taxonomies like the MLCommons Proof of Concept evaluation to facilitate collaboration and transparency on safety and content evaluations. Our Purple Llama tools are open sourced for the community to use and widely distributed across ecosystem partners including cloud service providers. We encourage community contributions to our [Github repository](https://github.com/meta-llama/PurpleLlama). + +Grants: We also set up the [Llama Impact Grants](https://llama.meta.com/llama-impact-grants/) program to identify and support the most compelling applications of Meta’s Llama model for societal benefit across three categories: education, climate and open innovation. The 20 finalists from the hundreds of applications can be found [here](https://llama.meta.com/llama-impact-grants/#finalists). + +Reporting: Finally, we put in place a set of resources including an [output reporting mechanism](https://developers.facebook.com/llama_output_feedback) and [bug bounty program](https://www.facebook.com/whitehat) to continuously improve the Llama technology with the help of the community. + +## Ethical Considerations and Limitations + +Values: The core values of Llama 3.2 are openness, inclusivity and helpfulness. It is meant to serve everyone, and to work for a wide range of use cases. It is thus designed to be accessible to people across many different backgrounds, experiences and perspectives. Llama 3.2 addresses users and their needs as they are, without insertion unnecessary judgment or normativity, while reflecting the understanding that even content that may appear problematic in some cases can serve valuable purposes in others. It respects the dignity and autonomy of all users, especially in terms of the values of free thought and expression that power innovation and progress. + +Testing: But Llama 3.2 is a new technology, and like any new technology, there are risks associated with its use. Testing conducted to date has not covered, nor could it cover, all scenarios. For these reasons, as with all LLMs, Llama 3.2’s potential outputs cannot be predicted in advance, and the model may in some instances produce inaccurate, biased or other objectionable responses to user prompts. Therefore, before deploying any applications of Llama 3.2 models, developers should perform safety testing and tuning tailored to their specific applications of the model. Please refer to available resources including our [Responsible Use Guide](https://llama.meta.com/responsible-use-guide), [Trust and Safety](https://llama.meta.com/trust-and-safety/) solutions, and other [resources](https://llama.meta.com/docs/get-started/) to learn more about responsible development. diff --git a/examples/DocQA/requirements.txt b/examples/DocQA/requirements.txt new file mode 100644 index 000000000..d680aa671 --- /dev/null +++ b/examples/DocQA/requirements.txt @@ -0,0 +1,5 @@ +llama-stack==0.0.57 +lxml>=4.9.0,<5.0.0 +xmltodict>=0.12.0,<1.0.0 +pycryptodomex>=3.8.2,<4.0.0 +google-cloud-storage>=1.16,<2.0 \ No newline at end of file diff --git a/examples/DocQA/scripts/caption_outputs.py b/examples/DocQA/scripts/caption_outputs.py new file mode 100644 index 000000000..8ab5cdf01 --- /dev/null +++ b/examples/DocQA/scripts/caption_outputs.py @@ -0,0 +1,127 @@ +import os +import asyncio +import argparse +import base64 +import mimetypes +from pathlib import Path +from llama_stack_client import LlamaStackClient +from llama_stack_client.lib.inference.event_logger import EventLogger + +def parse_args(): + parser = argparse.ArgumentParser(description='Process document images with LlamaStack Vision API') + parser.add_argument('--host', type=str, default='localhost', help='LlamaStack server host (default: localhost)') + parser.add_argument('--port', type=int, default=5000, help='LlamaStack server port (default: 5000)') + parser.add_argument('--input_dir', type=str, required=True, help='Input directory containing markdown files and images') + return parser.parse_args() + +def encode_image_to_data_url(file_path: str) -> str: + """Encode an image file to a data URL.""" + mime_type, _ = mimetypes.guess_type(file_path) + if mime_type is None: + raise ValueError("Could not determine MIME type of the file") + + with open(file_path, "rb") as image_file: + encoded_string = base64.b64encode(image_file.read()).decode("utf-8") + + return f"data:{mime_type};base64,{encoded_string}" + +class DocumentProcessor: + def __init__(self, host: str, port: int): + self.client = LlamaStackClient(base_url=f"http://{host}:{port}") + self.processed_images = {} + + async def get_image_caption(self, image_path: str) -> str: + """Get caption for an image using LlamaStack Vision API.""" + if image_path in self.processed_images: + return self.processed_images[image_path] + + try: + data_url = encode_image_to_data_url(image_path) + + message = { + "role": "user", + "content": [ + {"image": {"uri": data_url}}, + "This image comes from a scan inside a document, please provide a high level caption of what you see inside the image." + ] + } + + response = await self.client.inference.chat_completion( + messages=[message], + model="Llama3.2-11B-Vision-Instruct", + stream=False + ) + + caption = response.choices[0].message.content + self.processed_images[image_path] = caption + return caption + + except Exception as e: + print(f"Error processing image {image_path}: {str(e)}") + return None + + async def process_markdown_file(self, output_dir: str, md_filename: str) -> None: + """Process a single markdown file and replace image placeholders with captions.""" + print(f"Processing: {md_filename}") + + md_path = Path(output_dir) / md_filename + images_dir = Path(output_dir) / 'images' + + try: + content = md_path.read_text(encoding='utf-8') + except Exception as e: + print(f"Failed to read {md_filename}: {str(e)}") + return + + base_name = md_filename.rsplit('.', 1)[0] + image_count = 1 + updated = False + + while '' in content: + image_filename = f"{base_name}-figure-{image_count}.png" + image_path = images_dir / image_filename + + if not image_path.exists(): + print(f"Image not found: {image_filename}") + break + + caption = await self.get_image_caption(str(image_path)) + if caption: + image_markdown = f"![{caption}](images/{image_filename})\n\n_{caption}_" + content = content.replace('', image_markdown, 1) + print(f"Processed image {image_count} for {base_name}") + updated = True + else: + print(f"Failed to process image {image_filename}") + break + + image_count += 1 + + if updated: + try: + md_path.write_text(content, encoding='utf-8') + except Exception as e: + print(f"Failed to write updated content to {md_filename}: {str(e)}") + +async def main(): + args = parse_args() + output_dir = Path(args.input_dir) + + if not output_dir.exists(): + print(f"Input directory not found: {output_dir}") + return + + processor = DocumentProcessor(host=args.host, port=args.port) + md_files = list(output_dir.glob('*.md')) + + if not md_files: + print(f"No markdown files found in {output_dir}") + return + + for md_file in md_files: + await processor.process_markdown_file(output_dir, md_file.name) + + print("Processing completed") + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/DocQA/scripts/ingest_files.py b/examples/DocQA/scripts/ingest_files.py new file mode 100644 index 000000000..89252d987 --- /dev/null +++ b/examples/DocQA/scripts/ingest_files.py @@ -0,0 +1,182 @@ +import argparse +import json +import logging +import shutil +from pathlib import Path +from typing import List, Tuple + +import yaml +import os + +from docling.backend.pypdfium2_backend import PyPdfiumDocumentBackend +from docling.datamodel.base_models import InputFormat +from docling.datamodel.document import PictureItem, TableItem +from docling.datamodel.pipeline_options import PdfPipelineOptions +from docling.document_converter import ( + DocumentConverter, + PdfFormatOption, + WordFormatOption, +) +from docling.pipeline.simple_pipeline import SimplePipeline +from docling.pipeline.standard_pdf_pipeline import StandardPdfPipeline + +def parse_args(): + parser = argparse.ArgumentParser( + description="Process documents from input directory" + ) + parser.add_argument( + "--input_dir", + type=str, + required=True, + help="Input directory containing documents", + ) + parser.add_argument( + "--output_dir", + type=str, + help="Output directory for processed files (default: input_dir/output)", + ) + return parser.parse_args() + + +def get_document_files(input_dir: Path) -> Tuple[List[Path], List[Path]]: + """ + Recursively scan directory for document files. + Returns: + tuple: (documents_to_process, markdown_files) + """ + process_extensions = {".pdf", ".docx", ".pptx"} + documents_to_process = [] + markdown_files = [] + + for path in input_dir.rglob("*"): + if path.is_file(): + if path.suffix.lower() in process_extensions: + documents_to_process.append(path) + elif path.suffix.lower() == ".md": + markdown_files.append(path) + + return documents_to_process, markdown_files + + +def save_images(res, output_subdir: Path, doc_filename: str) -> List[Tuple[str, Path]]: + """ + Extracts and saves images from the document. + Returns a list of (image_type, image_path) tuples for future processing. + """ + images_dir = output_subdir / "images" + images_dir.mkdir(parents=True, exist_ok=True) + saved_images = [] + + # Save page images + for page_no, page in res.document.pages.items(): + if hasattr(page, "image") and page.image: + image_path = images_dir / f"{doc_filename}-page-{page_no}.png" + with image_path.open("wb") as fp: + page.image.pil_image.save(fp, format="PNG") + saved_images.append(("page", image_path)) + + # Save images of figures and tables + table_counter = 0 + picture_counter = 0 + + for element, _level in res.document.iterate_items(): + if ( + isinstance(element, TableItem) + and hasattr(element, "image") + and element.image + ): + table_counter += 1 + image_path = images_dir / f"{doc_filename}-table-{table_counter}.png" + with image_path.open("wb") as fp: + element.image.pil_image.save(fp, "PNG") + saved_images.append(("table", image_path)) + + if ( + isinstance(element, PictureItem) + and hasattr(element, "image") + and element.image + ): + picture_counter += 1 + image_path = images_dir / f"{doc_filename}-figure-{picture_counter}.png" + with image_path.open("wb") as fp: + element.image.pil_image.save(fp, "PNG") + saved_images.append(("figure", image_path)) + + return saved_images + + +def main(): + args = parse_args() + + # Set up input and output directories + input_dir = Path(args.input_dir) + output_dir = Path(args.output_dir) if args.output_dir else input_dir / "output" + if os.path.exists(output_dir): + print(f"Output directory already exists: {output_dir}, will stop ingestion") + raise Exception(f"Output directory already exists: {output_dir}") + else: + output_dir.mkdir(parents=True, exist_ok=True) + + # Get all document files recursively + documents_to_process, markdown_files = get_document_files(input_dir) + + # Copy markdown files directly + for md_file in markdown_files: + relative_path = md_file.relative_to(input_dir) + output_path = output_dir / relative_path + output_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(md_file, output_path) + print(f"Copied: {md_file}") + + if documents_to_process: + # Configure pipeline options + pipeline_options = PdfPipelineOptions() + pipeline_options.do_ocr = False + pipeline_options.images_scale = 2.0 + pipeline_options.generate_page_images = False + pipeline_options.generate_table_images = False + pipeline_options.generate_picture_images = True + + # Configure document converter + doc_converter = DocumentConverter( + allowed_formats=[ + InputFormat.PDF, + InputFormat.DOCX, + InputFormat.PPTX, + ], + format_options={ + InputFormat.PDF: PdfFormatOption( + pipeline_cls=StandardPdfPipeline, + backend=PyPdfiumDocumentBackend, + pipeline_options=pipeline_options, + ), + InputFormat.DOCX: WordFormatOption(pipeline_cls=SimplePipeline), + }, + ) + + # Process all documents + conv_results = doc_converter.convert_all(documents_to_process) + all_extracted_images = [] + + # Save results + for res in conv_results: + relative_path = res.input.file.relative_to(input_dir) + output_subdir = output_dir / relative_path.parent + output_subdir.mkdir(parents=True, exist_ok=True) + + md_path = output_subdir / f"{res.input.file.stem}.md" + + print(f"Converting: {res.input.file}" f"\nSaving to: {md_path}") + + extracted_images = save_images(res, output_subdir, res.input.file.stem) + all_extracted_images.extend(extracted_images) + + with md_path.open("w", encoding="utf-8") as fp: + fp.write(res.document.export_to_markdown()) + + print(f"\nExtracted {len(all_extracted_images)} images in total") + print("Ready for image captioning processing") + + +if __name__ == "__main__": + main() diff --git a/examples/agents/rag_with_memory_bank.py b/examples/agents/rag_with_memory_bank.py index 76f82884e..1dde56d19 100644 --- a/examples/agents/rag_with_memory_bank.py +++ b/examples/agents/rag_with_memory_bank.py @@ -36,22 +36,15 @@ async def run_main(host: str, port: int, disable_safety: bool = False): client = LlamaStackClient(base_url=f"http://{host}:{port}") providers = client.providers.list() - - available_shields = [shield.identifier for shield in client.shields.list()] - if not available_shields: - print("No available shields. Disable safety.") - else: - print(f"Available shields found: {available_shields}") - # create a memory bank client.memory_banks.register( - memory_bank_id="test_bank", - params={ + memory_bank={ + "identifier": "test_bank", "embedding_model": "all-MiniLM-L6-v2", "chunk_size_in_tokens": 512, "overlap_size_in_tokens": 64, - }, - provider_id=providers["memory"][0].provider_id, + "provider_id": providers["memory"][0].provider_id, + } ) # insert some documents @@ -67,7 +60,7 @@ async def run_main(host: str, port: int, disable_safety: bool = False): print(f"Using model: {selected_model}") agent_config = AgentConfig( - model=selected_model, + model="Llama3.1-8B-Instruct", instructions="You are a helpful assistant", sampling_params={ "strategy": "greedy", @@ -121,4 +114,4 @@ def main(host: str, port: int): if __name__ == "__main__": - fire.Fire(main) + fire.Fire(main) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 90082937a..f84ea4bee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,5 @@ -fastapi fire -httpx mesop python-dotenv setuptools -uvicorn yfinance -gradio -llama-stack>=0.0.57 -llama-stack-client>=0.0.57