From ee89ce4997dc45a799c320226768a70a0c6b0343 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 12:46:53 +0200 Subject: [PATCH] chore: sync content to repo (#10209) Co-authored-by: nilbuild <4921183+nilbuild@users.noreply.github.com> --- .../content/arize-ai@jx57heeBnacrlCyQPhlEi.md | 8 ++++++++ .../content/constraints@PT3uDiUjiKhO8laOkCmgP.md | 7 +++++++ .../costlatency-monitoring@CG3XTMPug98s-EP0_rn1Y.md | 8 ++++++++ .../content/deepeval@hyN5v8dg1lHJLpdZZdtJE.md | 9 +++++++++ .../deterministic-evals@kSDTJK74Tq0KHBfIawJuT.md | 8 ++++++++ .../evaluation-metrics@Z8MBleL51wJuV9BkS1djI.md | 8 ++++++++ .../ai-engineer/content/gemma@MNtaY1_kOJHeoWuM-abb4.md | 2 +- .../content/helicone@82LE-qxw1H_FYfVUdgkN9.md | 8 ++++++++ .../content/human-evals@QIx5nT75l2Cn2q75-TxfD.md | 8 ++++++++ .../content/langfuse@Xw6-crHimGrVC1u1xujc6.md | 8 ++++++++ .../content/langsmith@bN_ReFUQgkJmWOAFMLXMg.md | 8 ++++++++ .../large-language-model-llm@wf2BSyUekr1S1q6l8kyq6.md | 2 +- .../content/llm-evaluations@FJ3ZwbTdiFyebrFi96tnz.md | 8 ++++++++ .../content/llm-observability@1bBgMVISENC-XBwuZFlUk.md | 8 ++++++++ .../content/model-based-evals@LJ6_U5lOdc0uRngutj7Ro.md | 9 +++++++++ .../content/openai-vision-api@CRrqa-dBw1LlOwVbrZhjK.md | 2 +- .../production-monitoring@3gwVG4XaO83z3Yzdd84Z-.md | 7 +++++++ .../prompt-engineering@ozrR8IvjNFbHd44kZrExX.md | 10 ++++++++++ .../rag--dynamic-filters@LnQ2AatMWpExUHcZhDIPd.md | 9 +++++++++ .../ai-engineer/content/ragas@CZ_-PSDzzSrSon5XdzLen.md | 9 +++++++++ .../ai-engineer/content/rags@IX1BJWGwGmB4L063g0Frf.md | 9 +++++++++ .../regression-testing@Bkzi3QyzKyHxcHE7sodRZ.md | 8 ++++++++ .../content/tracing--logging@cdapQf0Owxdx6olAHTrbq.md | 7 +++++++ 23 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 roadmaps/ai-engineer/content/arize-ai@jx57heeBnacrlCyQPhlEi.md create mode 100644 roadmaps/ai-engineer/content/constraints@PT3uDiUjiKhO8laOkCmgP.md create mode 100644 roadmaps/ai-engineer/content/costlatency-monitoring@CG3XTMPug98s-EP0_rn1Y.md create mode 100644 roadmaps/ai-engineer/content/deepeval@hyN5v8dg1lHJLpdZZdtJE.md create mode 100644 roadmaps/ai-engineer/content/deterministic-evals@kSDTJK74Tq0KHBfIawJuT.md create mode 100644 roadmaps/ai-engineer/content/evaluation-metrics@Z8MBleL51wJuV9BkS1djI.md create mode 100644 roadmaps/ai-engineer/content/helicone@82LE-qxw1H_FYfVUdgkN9.md create mode 100644 roadmaps/ai-engineer/content/human-evals@QIx5nT75l2Cn2q75-TxfD.md create mode 100644 roadmaps/ai-engineer/content/langfuse@Xw6-crHimGrVC1u1xujc6.md create mode 100644 roadmaps/ai-engineer/content/langsmith@bN_ReFUQgkJmWOAFMLXMg.md create mode 100644 roadmaps/ai-engineer/content/llm-evaluations@FJ3ZwbTdiFyebrFi96tnz.md create mode 100644 roadmaps/ai-engineer/content/llm-observability@1bBgMVISENC-XBwuZFlUk.md create mode 100644 roadmaps/ai-engineer/content/model-based-evals@LJ6_U5lOdc0uRngutj7Ro.md create mode 100644 roadmaps/ai-engineer/content/production-monitoring@3gwVG4XaO83z3Yzdd84Z-.md create mode 100644 roadmaps/ai-engineer/content/prompt-engineering@ozrR8IvjNFbHd44kZrExX.md create mode 100644 roadmaps/ai-engineer/content/rag--dynamic-filters@LnQ2AatMWpExUHcZhDIPd.md create mode 100644 roadmaps/ai-engineer/content/ragas@CZ_-PSDzzSrSon5XdzLen.md create mode 100644 roadmaps/ai-engineer/content/rags@IX1BJWGwGmB4L063g0Frf.md create mode 100644 roadmaps/ai-engineer/content/regression-testing@Bkzi3QyzKyHxcHE7sodRZ.md create mode 100644 roadmaps/ai-engineer/content/tracing--logging@cdapQf0Owxdx6olAHTrbq.md diff --git a/roadmaps/ai-engineer/content/arize-ai@jx57heeBnacrlCyQPhlEi.md b/roadmaps/ai-engineer/content/arize-ai@jx57heeBnacrlCyQPhlEi.md new file mode 100644 index 000000000..7e5d6cfe2 --- /dev/null +++ b/roadmaps/ai-engineer/content/arize-ai@jx57heeBnacrlCyQPhlEi.md @@ -0,0 +1,8 @@ +# Arize AI + +Arize is an ML observability platform that supports both traditional ML models and LLM applications. For LLMs, it provides tracing, drift detection, and evaluation tooling. It is often used in enterprise settings where you need to monitor models already deployed in production and need deep integration with existing ML infrastructure. + +Visit the following resources to learn more: + +- [@official@Arize AI Docs](https://arize.com/docs/ax) +- [@video@How to Evaluate Tool-Calling Agents](https://www.youtube.com/watch?v=JytjrbDaI44&list=PL86ARIu_ElO5F3LCwM-k3ZDhvnuigxqbw) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/constraints@PT3uDiUjiKhO8laOkCmgP.md b/roadmaps/ai-engineer/content/constraints@PT3uDiUjiKhO8laOkCmgP.md new file mode 100644 index 000000000..3e989bc0e --- /dev/null +++ b/roadmaps/ai-engineer/content/constraints@PT3uDiUjiKhO8laOkCmgP.md @@ -0,0 +1,7 @@ +# Constraining Prompts + +Constraining system prompts involves explicitly defining boundaries and limitations within the instructions given to a large language model (LLM). This technique focuses on guiding the LLM to operate within a specific scope, preventing it from generating irrelevant, harmful, or factually incorrect responses by setting rules and restrictions on its behavior and output format. + +Visit the following resources to learn more: + +- [@article@Introduction: The Power of Clear Instructions](https://codesignal.com/learn/courses/prompting-foundations/lessons/defining-constraints-and-requirements-for-effective-prompts) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/costlatency-monitoring@CG3XTMPug98s-EP0_rn1Y.md b/roadmaps/ai-engineer/content/costlatency-monitoring@CG3XTMPug98s-EP0_rn1Y.md new file mode 100644 index 000000000..3be145682 --- /dev/null +++ b/roadmaps/ai-engineer/content/costlatency-monitoring@CG3XTMPug98s-EP0_rn1Y.md @@ -0,0 +1,8 @@ +# Cost & Latency Monitoring + +Cost and latency monitoring tracks token usage, the resulting financial cost, and response times across your AI system. Without this visibility, production costs can compound quickly and silently, especially when using large reasoning models that charge significantly per token. Tracking these alongside quality metrics lets you make informed tradeoffs, such as routing simpler queries to cheaper models or caching common responses to reduce redundant API calls. + +Visit the following resources to learn more: + +- [@article@The LLM Inference Trilemma: Throughput, Latency, Cost](https://www.digitalocean.com/blog/llm-inference-tradeoffs) +- [@article@LLM Cost Optimization: 5 Levers That Cut API Spend 70-85%](https://www.morphllm.com/llm-cost-optimization) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/deepeval@hyN5v8dg1lHJLpdZZdtJE.md b/roadmaps/ai-engineer/content/deepeval@hyN5v8dg1lHJLpdZZdtJE.md new file mode 100644 index 000000000..3a155fe2a --- /dev/null +++ b/roadmaps/ai-engineer/content/deepeval@hyN5v8dg1lHJLpdZZdtJE.md @@ -0,0 +1,9 @@ +# DeepEval + +DeepEval is an open-source LLM evaluation framework that lets you write automated tests for your AI application in a way that feels similar to unit testing with pytest. It comes with over 50 built-in metrics covering accuracy, faithfulness, relevance, toxicity, and agent behavior, and it works across RAG pipelines, chatbots, and multi-step agents. Because it integrates directly into CI/CD pipelines, teams can run evals automatically on every code change rather than treating evaluation as a manual, one-off task. + +Visit the following resources to learn more: + +- [@official@DeepEval Docs](https://deepeval.com/docs/introduction) +- [@opensource@deepeval](https://github.com/confident-ai/deepeval) +- [@video@How to Setup DeepEval for Fast, Easy, and Powerful LLM Evaluations](https://www.youtube.com/watch?v=7YgpH5PauRM) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/deterministic-evals@kSDTJK74Tq0KHBfIawJuT.md b/roadmaps/ai-engineer/content/deterministic-evals@kSDTJK74Tq0KHBfIawJuT.md new file mode 100644 index 000000000..9a6102171 --- /dev/null +++ b/roadmaps/ai-engineer/content/deterministic-evals@kSDTJK74Tq0KHBfIawJuT.md @@ -0,0 +1,8 @@ +# Deterministic Evals + +Deterministic evals use fixed, rule-based checks to score model outputs. Examples include checking whether a response contains a required keyword, matches an expected format, or equals a known correct answer exactly. They are fast, cheap, and fully reproducible, making them a good first layer of evaluation for any system. Their limitation is that they cannot assess open-ended or subjective output quality. + +Visit the following resources to learn more: + +- [@article@How I Built Deterministic LLM Evaluation Metrics for DeepEval](https://www.confident-ai.com/blog/how-i-built-deterministic-llm-evaluation-metrics-for-deepeval) +- [@article@Demystifying evals for AI agents](https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/evaluation-metrics@Z8MBleL51wJuV9BkS1djI.md b/roadmaps/ai-engineer/content/evaluation-metrics@Z8MBleL51wJuV9BkS1djI.md new file mode 100644 index 000000000..0796a2d7d --- /dev/null +++ b/roadmaps/ai-engineer/content/evaluation-metrics@Z8MBleL51wJuV9BkS1djI.md @@ -0,0 +1,8 @@ +# Evaluation MetricsEvaluation Metrics + +Evaluation metrics are the specific measurements used to score LLM outputs against defined quality criteria. Each metric targets a different dimension of quality, such as whether the response is factually grounded, relevant to the question, free of harmful content, or genuinely useful to the user. Choosing the right set of metrics for your use case is one of the most important decisions in building an eval system, since metrics determine what your system optimizes for and what failure modes it can actually detect. + +Visit the following resources to learn more: + +- [@article@LLM evaluation metrics and methods, explained simply](https://www.evidentlyai.com/llm-guide/llm-evaluation-metrics) +- [@article@A list of metrics for evaluating LLM-generated content](https://learn.microsoft.com/en-us/ai/playbook/technology-guidance/generative-ai/working-with-llms/evaluation/list-of-eval-metrics) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/gemma@MNtaY1_kOJHeoWuM-abb4.md b/roadmaps/ai-engineer/content/gemma@MNtaY1_kOJHeoWuM-abb4.md index d300d68aa..686dd795d 100644 --- a/roadmaps/ai-engineer/content/gemma@MNtaY1_kOJHeoWuM-abb4.md +++ b/roadmaps/ai-engineer/content/gemma@MNtaY1_kOJHeoWuM-abb4.md @@ -4,4 +4,4 @@ Gemma is a family of open-source large language models (LLMs) developed by Googl Visit the following resources to learn more: -- [@official@Gemma](https://deepmind.google/models/gemma/) +- [@official@Gemma](https://deepmind.google/models/gemma/) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/helicone@82LE-qxw1H_FYfVUdgkN9.md b/roadmaps/ai-engineer/content/helicone@82LE-qxw1H_FYfVUdgkN9.md new file mode 100644 index 000000000..5afcb0290 --- /dev/null +++ b/roadmaps/ai-engineer/content/helicone@82LE-qxw1H_FYfVUdgkN9.md @@ -0,0 +1,8 @@ +# Helicone + +Helicone is a logging and observability proxy for LLM APIs. Instead of calling OpenAI or Anthropic directly, you route requests through Helicone, which captures every request and response with zero code changes. It provides dashboards for cost tracking, latency, error rates, and user-level analytics. + +Visit the following resources to learn more: + +- [@official@Helicone Docs](https://docs.helicone.ai/getting-started/quick-start) +- [@video@ES Skip navigation Helicone tutorial Create Avatar image Helicone AI — The Open-source LLM Observability for Developers](https://www.youtube.com/watch?v=RNFa8bl3RdE) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/human-evals@QIx5nT75l2Cn2q75-TxfD.md b/roadmaps/ai-engineer/content/human-evals@QIx5nT75l2Cn2q75-TxfD.md new file mode 100644 index 000000000..297c9e37a --- /dev/null +++ b/roadmaps/ai-engineer/content/human-evals@QIx5nT75l2Cn2q75-TxfD.md @@ -0,0 +1,8 @@ +# Human Evals + +Human evals involve people directly reviewing and scoring model outputs against defined criteria. They are the most accurate form of evaluation for nuanced or subjective quality dimensions, and serve as the ground truth that other eval methods are validated against. Human evals are slower and more expensive than automated approaches, so they are typically used for high-stakes decisions, calibrating automated evals, or reviewing edge cases flagged by other methods. + +Visit the following resources to learn more: + +- [@article@LLM-as-a-judge vs. human evaluation: Why together is better](https://www.superannotate.com/blog/llm-as-a-judge-vs-human-evaluation) +- [@article@How to run human-in-the-loop evals for LLM apps](https://www.braintrust.dev/articles/human-in-the-loop-evals-for-llm-apps) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/langfuse@Xw6-crHimGrVC1u1xujc6.md b/roadmaps/ai-engineer/content/langfuse@Xw6-crHimGrVC1u1xujc6.md new file mode 100644 index 000000000..8fbd7b07d --- /dev/null +++ b/roadmaps/ai-engineer/content/langfuse@Xw6-crHimGrVC1u1xujc6.md @@ -0,0 +1,8 @@ +# LangFuse + +LangFuse is an open-source LLM observability platform that provides tracing, prompt management, and evaluation tooling. You can self-host it or use the cloud version. It integrates with most popular frameworks and SDKs through a simple API, and lets you score traces manually or automatically to build up a quality signal over time. + +Visit the following resources to learn more: + +- [@official@Langfuse Overview](https://langfuse.com/docs) +- [@video@Get Started with Langfuse - Open-Source LLM Monitoring](https://www.youtube.com/watch?v=epnPfe5am3I&list=PL4v4zZF1rGG5slCdloT33O7Qe_Zq1zHfo) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/langsmith@bN_ReFUQgkJmWOAFMLXMg.md b/roadmaps/ai-engineer/content/langsmith@bN_ReFUQgkJmWOAFMLXMg.md new file mode 100644 index 000000000..434a3af6c --- /dev/null +++ b/roadmaps/ai-engineer/content/langsmith@bN_ReFUQgkJmWOAFMLXMg.md @@ -0,0 +1,8 @@ +# LangSmith + +LangSmith is an observability and evaluation platform built by the LangChain team, designed specifically for LLM applications. It automatically captures traces of LangChain runs and also works with non-LangChain code. It lets you inspect inputs and outputs at every step of a chain or agent, compare prompt versions, and run evals directly on logged traces. + +Visit the following resources to learn more: + +- [@official@LagLangSmith docsnS](https://docs.langchain.com/langsmith/home) +- [@video@Why Evals Matter | LangSmith Evaluations](https://www.youtube.com/watch?v=vygFgCNR7WA&list=PLfaIDFEXuae0um8Fj0V4dHG37fGFU8Q5S) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/large-language-model-llm@wf2BSyUekr1S1q6l8kyq6.md b/roadmaps/ai-engineer/content/large-language-model-llm@wf2BSyUekr1S1q6l8kyq6.md index 4a7c77a65..03ec7db5b 100644 --- a/roadmaps/ai-engineer/content/large-language-model-llm@wf2BSyUekr1S1q6l8kyq6.md +++ b/roadmaps/ai-engineer/content/large-language-model-llm@wf2BSyUekr1S1q6l8kyq6.md @@ -1,6 +1,6 @@ # LLMs -LLMs, or Large Language Models, are advanced AI models trained on vast datasets to understand and generate human-like text. They can perform a wide range of natural language processing tasks, such as text generation, translation, summarization, and question answering. Examples include GPT-5, BERT, and DeepSeek. LLMs are capable of understanding context, handling complex queries, and generating coherent responses, making them useful for applications like chatbots, content creation, and automated support. However, they require significant computational resources and may carry biases from their training data. +An LLM (Large Language Model) is an AI trained on vast text datasets to understand and generate human-like language via next-token prediction. While it natively handles tasks like translation, summarization, and writing, capabilities like web browsing or code execution come from software layers wrapped around the model. Though highly versatile, LLMs require massive computing power and can mirror biases present in their training data. Visit the following resources to learn more: diff --git a/roadmaps/ai-engineer/content/llm-evaluations@FJ3ZwbTdiFyebrFi96tnz.md b/roadmaps/ai-engineer/content/llm-evaluations@FJ3ZwbTdiFyebrFi96tnz.md new file mode 100644 index 000000000..336bfafeb --- /dev/null +++ b/roadmaps/ai-engineer/content/llm-evaluations@FJ3ZwbTdiFyebrFi96tnz.md @@ -0,0 +1,8 @@ +# LLM Evaluations + +LLM evaluations are structured tests that measure how well a model or AI system performs across a set of defined criteria. Rather than judging outputs informally, evals give you a repeatable, quantitative way to compare prompt versions, model updates, and system changes. They are the primary tool for making confident, evidence-based decisions about your AI system. + +Visit the following resources to learn more: + +- [@article@LLM evaluation: a beginner's guide](https://www.evidentlyai.com/llm-guide/llm-evaluation) +- [@video@Welcome to the LLM evaluation course](https://www.youtube.com/watch?v=rHs0sP7b5fM&list=PL9omX6impEuMgDFCK_NleIB0sMzKs2boI) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/llm-observability@1bBgMVISENC-XBwuZFlUk.md b/roadmaps/ai-engineer/content/llm-observability@1bBgMVISENC-XBwuZFlUk.md new file mode 100644 index 000000000..06d86460f --- /dev/null +++ b/roadmaps/ai-engineer/content/llm-observability@1bBgMVISENC-XBwuZFlUk.md @@ -0,0 +1,8 @@ +# LLM Observability + +LLM observability is the practice of monitoring and understanding what happens inside your AI application at runtime, tracking things like which prompts were sent, what responses came back, how long each call took, and how many tokens were used. Without visibility into these details, debugging failures or understanding why outputs changed is extremely difficult. Good observability gives you a continuous record of your system's behavior in development and production. + +Visit the following resources to learn more: + +- [@article@What is LLM observability?](https://www.ibm.com/think/topics/llm-observability) +- [@article@LLM Evaluation and AI Observability for Agent Monitoring](https://blog.jetbrains.com/pycharm/2026/05/llm-evaluation-and-ai-observability-for-agent-monitoring/) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/model-based-evals@LJ6_U5lOdc0uRngutj7Ro.md b/roadmaps/ai-engineer/content/model-based-evals@LJ6_U5lOdc0uRngutj7Ro.md new file mode 100644 index 000000000..874e8b8af --- /dev/null +++ b/roadmaps/ai-engineer/content/model-based-evals@LJ6_U5lOdc0uRngutj7Ro.md @@ -0,0 +1,9 @@ +# Model-Based Evals + +Model-based evals use a separate AI model to automatically score or assess the outputs of your LLM application. Instead of writing manual rules or relying on human reviewers, you delegate the judgment to another model, a technique commonly known as LLM-as-a-Judge. You write a prompt describing the evaluation criteria, and the judge model rates the response. This approach handles subjective, open-ended quality dimensions that rules cannot capture, while scaling far more cheaply than human review, though it requires careful prompt design to avoid bias and inconsistency in the judges themselves. + +Visit the following resources to learn more: + +- [@article@A pragmatic guide to LLM evals for devs](https://newsletter.pragmaticengineer.com/p/evals) +- [@article@LLM-as-a-judge: a complete guide to using LLMs for evaluations](https://www.evidentlyai.com/llm-guide/llm-as-a-judge) +- [@video@LLM as a Judge: Scaling AI Evaluation Strategies](https://www.youtube.com/watch?v=trfUBIDeI1Y) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/openai-vision-api@CRrqa-dBw1LlOwVbrZhjK.md b/roadmaps/ai-engineer/content/openai-vision-api@CRrqa-dBw1LlOwVbrZhjK.md index 86942eb05..3845f770c 100644 --- a/roadmaps/ai-engineer/content/openai-vision-api@CRrqa-dBw1LlOwVbrZhjK.md +++ b/roadmaps/ai-engineer/content/openai-vision-api@CRrqa-dBw1LlOwVbrZhjK.md @@ -5,4 +5,4 @@ The OpenAI Vision API enables models to analyze and understand images, allowing Visit the following resources to learn more: - [@official@Vision](https://developers.openai.com/api/docs/guides/images-vision) -- [@video@OpenAI Vision API Crash Course](https://www.youtube.com/watch?v=ZjkS11DSeEk) +- [@video@OpenAI Vision API Crash Course](https://www.youtube.com/watch?v=ZjkS11DSeEk) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/production-monitoring@3gwVG4XaO83z3Yzdd84Z-.md b/roadmaps/ai-engineer/content/production-monitoring@3gwVG4XaO83z3Yzdd84Z-.md new file mode 100644 index 000000000..3f8910ab7 --- /dev/null +++ b/roadmaps/ai-engineer/content/production-monitoring@3gwVG4XaO83z3Yzdd84Z-.md @@ -0,0 +1,7 @@ +# Production Monitoring + +Production monitoring is the continuous observation of your AI system once it is live and handling real user traffic. Unlike testing in a controlled environment, production surfaces edge cases, unexpected inputs, and failure modes that never appeared during development. Monitoring in production means tracking quality metrics, error rates, and behavioral changes over time so that regressions and anomalies are caught before they affect a large number of users. + +Visit the following resources to learn more: + +- [@article@LLM Observability: The Complete Guide to Monitoring LLMs in Production](https://www.swept.ai/post/llm-observability-complete-guide) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/prompt-engineering@ozrR8IvjNFbHd44kZrExX.md b/roadmaps/ai-engineer/content/prompt-engineering@ozrR8IvjNFbHd44kZrExX.md new file mode 100644 index 000000000..1f1e7a335 --- /dev/null +++ b/roadmaps/ai-engineer/content/prompt-engineering@ozrR8IvjNFbHd44kZrExX.md @@ -0,0 +1,10 @@ +# Prompt vs. Context Engineering + +Prompt engineering focuses on crafting effective instructions (prompts) to elicit desired outputs from Large Language Models (LLMs). Context engineering, on the other hand, involves structuring and providing relevant background information (the context) to the LLM, enabling it to generate more accurate and informed responses based on a richer understanding of the subject matter. + +Visit the following resources to learn more: + +- [@article@Prompt engineering: overview and guide](https://cloud.google.com/discover/what-is-prompt-engineering?hl=en) +- [@article@What is prompt engineering?](https://www.ibm.com/think/topics/prompt-engineering) +- [@video@Google's 9 Hour AI Prompt Engineering Course In 20 Minutes](https://www.youtube.com/watch?v=p09yRj47kNM) +- [@video@Prompt Engineering Full Course](https://www.youtube.com/watch?v=2BpCk4d2Cc0) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/rag--dynamic-filters@LnQ2AatMWpExUHcZhDIPd.md b/roadmaps/ai-engineer/content/rag--dynamic-filters@LnQ2AatMWpExUHcZhDIPd.md new file mode 100644 index 000000000..b0a13a1a1 --- /dev/null +++ b/roadmaps/ai-engineer/content/rag--dynamic-filters@LnQ2AatMWpExUHcZhDIPd.md @@ -0,0 +1,9 @@ +# RAG and Dynamic Filters + +Retrieval-Augmented Generation (RAG) enhances Large Language Models (LLMs) by providing them with relevant, up-to-date information from external sources. Dynamic filters are techniques that selectively filter the information retrieved for RAG, ensuring that the LLM receives only the most pertinent context based on the specific query and user. This results in more accurate, focused, and contextually appropriate LLM responses. + +Visit the following resources to learn more: + +- [@article@4 context engineering strategies every AI engineer needs to know](https://newsletter.owainlewis.com/p/4-context-engineering-strategies) +- [@article@Context Engineering](https://blog.langchain.com/context-engineering-for-agents/) +- [@article@Is RAG Dead? The Rise of Context Engineering and Semantic Layers for Agentic AI](https://towardsdatascience.com/beyond-rag/?utm_source=roadmap&utm_medium=Referral&utm_campaign=TDS+roadmap+integration) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/ragas@CZ_-PSDzzSrSon5XdzLen.md b/roadmaps/ai-engineer/content/ragas@CZ_-PSDzzSrSon5XdzLen.md new file mode 100644 index 000000000..613f5832f --- /dev/null +++ b/roadmaps/ai-engineer/content/ragas@CZ_-PSDzzSrSon5XdzLen.md @@ -0,0 +1,9 @@ +# RAGAS + +RAGAS is an open-source framework specifically designed for evaluating retrieval-augmented generation pipelines. It provides a suite of reference-free metrics, meaning it does not require hand-labeled ground truth answers, which makes it practical to run on real production data. Its core metrics cover the main ways a RAG system can fail: retrieving the wrong context, generating answers that are not supported by the retrieved context, and producing responses that do not actually address the user's question. + +Visit the following resources to learn more: + +- [@official@Ragas Docs](https://docs.ragas.io/en/stable/) +- [@opensource@ragas](https://github.com/vibrantlabsai/ragas) +- [@video@RAGAS: How to Evaluate a RAG Application](https://www.youtube.com/watch?v=5fp6e5nhJRk) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/rags@IX1BJWGwGmB4L063g0Frf.md b/roadmaps/ai-engineer/content/rags@IX1BJWGwGmB4L063g0Frf.md new file mode 100644 index 000000000..ee10c2ca7 --- /dev/null +++ b/roadmaps/ai-engineer/content/rags@IX1BJWGwGmB4L063g0Frf.md @@ -0,0 +1,9 @@ +# RAG + +Retrieval-Augmented Generation (RAG) is an AI approach that combines information retrieval with language generation to create more accurate, contextually relevant outputs. It works by first retrieving relevant data from a knowledge base or external source, then using a language model to generate a response based on that information. This method enhances the accuracy of generative models by grounding their outputs in real-world data, making RAG ideal for tasks like question answering, summarization, and chatbots that require reliable, up-to-date information. + +Visit the following resources to learn more: + +- [@article@What is Retrieval-Augmented Generation? - Google](https://cloud.google.com/use-cases/retrieval-augmented-generation) +- [@article@RAG Explained: Understanding Embeddings, Similarity, and Retrieval](https://towardsdatascience.com/rag-explained-understanding-embeddings-similarity-and-retrieval/?utm_source=roadmap&utm_medium=Referral&utm_campaign=TDS+roadmap+integration) +- [@video@What is Retrieval-Augmented Generation? - IBM](https://www.youtube.com/watch?v=T-D1OfcDW1M) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/regression-testing@Bkzi3QyzKyHxcHE7sodRZ.md b/roadmaps/ai-engineer/content/regression-testing@Bkzi3QyzKyHxcHE7sodRZ.md new file mode 100644 index 000000000..bc5397790 --- /dev/null +++ b/roadmaps/ai-engineer/content/regression-testing@Bkzi3QyzKyHxcHE7sodRZ.md @@ -0,0 +1,8 @@ +# Regression Testing + +Regression testing for AI systems means verifying that a change to a prompt, model version, retrieval strategy, or any other component does not degrade performance on cases that previously worked correctly. Because LLM behavior can shift in subtle ways with small changes, having a regression suite lets you catch quality regressions before they reach users. + +Visit the following resources to learn more: + +- [@article@Watch the language: A tutorial on regression testing for LLMs](https://www.evidentlyai.com/blog/llm-regression-testing-tutorial) +- [@video@Regression Testing for LLMs: Golden Datasets Explained](https://www.youtube.com/watch?v=7vqU_Yj5kUc) \ No newline at end of file diff --git a/roadmaps/ai-engineer/content/tracing--logging@cdapQf0Owxdx6olAHTrbq.md b/roadmaps/ai-engineer/content/tracing--logging@cdapQf0Owxdx6olAHTrbq.md new file mode 100644 index 000000000..c6bd85b7c --- /dev/null +++ b/roadmaps/ai-engineer/content/tracing--logging@cdapQf0Owxdx6olAHTrbq.md @@ -0,0 +1,7 @@ +# Tracing & Logging + +Tracing records the full lifecycle of a request through your AI system, from the initial user input through any intermediate LLM calls, tool uses, or retrieval steps, all the way to the final response. Logging captures individual events like errors, latency spikes, or unexpected outputs. Together, they let you reconstruct exactly what happened during any given interaction, which is essential for debugging agents and multi-step pipelines. + +Visit the following resources to learn more: + +- [@article@A guide to LLM debugging, tracing, and monitoring](https://wandb.ai/onlineinference/genai-research/reports/A-guide-to-LLM-debugging-tracing-and-monitoring--VmlldzoxMzk1MjAyOQ) \ No newline at end of file