[{"@context":"https:\/\/schema.org\/","@type":"BlogPosting","@id":"https:\/\/blog.terabox.com\/insights\/build-large-language-models-from-scratch#BlogPosting","mainEntityOfPage":"https:\/\/blog.terabox.com\/insights\/build-large-language-models-from-scratch","headline":"Build Large Language Models (LLMs) from Scratch: A Guide","name":"Build Large Language Models (LLMs) from Scratch: A Guide","description":"\ud83d\udcfa Today’s recommended deep-dive video: https:\/\/www.youtube.com\/watch?v=quh7z1q7-uc Coding the Core: A Deep Dive into Building LLMs from ScratchThe Foundation of Tokens and DataArchitecture and the Generation LoopTraining and Efficient Fine-TuningKey TakeawaysQ&A Coding the Core: A Deep Dive into Building LLMs from... ","datePublished":"2026-08-13","dateModified":"2026-08-13","author":{"@type":"Person","@id":"https:\/\/blog.terabox.com\/author\/flextech-admin\/#Person","name":"flextech-admin","url":"https:\/\/blog.terabox.com\/author\/flextech-admin\/","image":{"@type":"ImageObject","@id":"https:\/\/secure.gravatar.com\/avatar\/ad516503a11cd5ca435acc9bb6523536?s=150&d=mm&r=gforcedefault=1","url":"https:\/\/secure.gravatar.com\/avatar\/ad516503a11cd5ca435acc9bb6523536?s=150&d=mm&r=gforcedefault=1","height":96,"width":96}},"publisher":{"@type":"Organization","name":"terabox","logo":{"@type":"ImageObject","@id":"http:\/\/blog.terabox.com\/wp-content\/uploads\/2021\/11\/logo\u4ea7\u54c1\u540d-\u7ad6\u7248.png","url":"http:\/\/blog.terabox.com\/wp-content\/uploads\/2021\/11\/logo\u4ea7\u54c1\u540d-\u7ad6\u7248.png","width":900,"height":900}},"image":{"@type":"ImageObject","@id":"https:\/\/img.youtube.com\/vi\/quh7z1q7-uc\/maxresdefault.jpg","url":"https:\/\/img.youtube.com\/vi\/quh7z1q7-uc\/maxresdefault.jpg","height":"","width":""},"url":"https:\/\/blog.terabox.com\/insights\/build-large-language-models-from-scratch","video":{"@context":"http:\/\/schema.org\/","@type":"VideoObject","@id":"https:\/\/www.youtube.com\/watch?v=quh7z1q7-uc#VideoObject","contentUrl":"https:\/\/www.youtube.com\/watch?v=quh7z1q7-uc","name":"Building LLMs from the Ground Up: A 3-hour Coding Workshop","description":"REFERENCES:\n1. Build an LLM from Scratch book: https:\/\/amzn.to\/4fqvn0D\n2. Build an LLM from Scratch repo: https:\/\/github.com\/rasbt\/LLMs-from-scratch\n3. GitHub repository with workshop code: https:\/\/github.com\/rasbt\/LLM-workshop-2024\n4. Lightning Studio for this workshop: https:\/\/lightning.ai\/lightning-ai\/studios\/llms-from-the-ground-up-workshop?view=public\n5. LitGPT: https:\/\/github.com\/Lightning-AI\/litgpt\n\nDESCRIPTION:\nThis tutorial is aimed at coders interested in understanding the building blocks of large language models (LLMs), how LLMs work, and how to code them from the ground up in PyTorch. We will kick off this tutorial with an introduction to LLMs, recent milestones, and their use cases. Then, we will code a small GPT-like LLM, including its data input pipeline, core architecture components, and pretraining code ourselves. After understanding how everything fits together and how to pretrain an LLM, we will learn how to load pretrained weights and finetune LLMs using open-source libraries.\n\n---\n\nTo support this channel, please consider purchasing a copy of my books: https:\/\/sebastianraschka.com\/books\/\n\n---\n\nhttps:\/\/twitter.com\/rasbt\nhttps:\/\/linkedin.com\/in\/sebastianraschka\/\nhttps:\/\/magazine.sebastianraschka.com\n\n---\n\nOUTLINE:\n0:00 \u2013 Workshop overview\n2:17 \u2013 Part 1: Intro to LLMs\n9:14 \u2013 Workshop materials\n10:48 \u2013 Part 2: Understanding LLM input data\n23:25 \u2013 A simple tokenizer class\n41:03 \u2013 Part 3: Coding an LLM architecture\n45:01 \u2013 GPT-2 and Llama 2\n1:07:11 \u2013 Part 4: Pretraining\n1:29:37 \u2013 Part 5.1: Loading pretrained weights\n1:45:12 \u2013 Part 5.2: Pretrained weights via LitGPT\n1:53:09 \u2013 Part 6.1: Instruction finetuning\n2:08:21 \u2013 Part 6.2: Instruction finetuning via LitGPT\n02:26:45 \u2013 Part 6.3: Benchmark evaluation\n02:36:55 \u2013 Part 6.4: Evaluating conversational performance\n02:42:40 \u2013 Conclusion","thumbnailUrl":["https:\/\/i.ytimg.com\/vi\/quh7z1q7-uc\/default.jpg","https:\/\/i.ytimg.com\/vi\/quh7z1q7-uc\/mqdefault.jpg","https:\/\/i.ytimg.com\/vi\/quh7z1q7-uc\/hqdefault.jpg","https:\/\/i.ytimg.com\/vi\/quh7z1q7-uc\/sddefault.jpg","https:\/\/i.ytimg.com\/vi\/quh7z1q7-uc\/maxresdefault.jpg"],"uploadDate":"2024-08-31T10:36:48+00:00","duration":"PT2H45M10S","embedUrl":"https:\/\/www.youtube.com\/embed\/quh7z1q7-uc","publisher":{"@type":"Organization","@id":"https:\/\/www.youtube.com\/channel\/UC_CzsS7UTjcxJ-xXp1ftxtA#Organization","url":"https:\/\/www.youtube.com\/channel\/UC_CzsS7UTjcxJ-xXp1ftxtA","name":"Sebastian Raschka","description":"ML\/AI researcher & former stats professor turned LLM research engineer. Author of \"Build a Large Language Model From Scratch\" (amzn.to\/4fqvn0D) & reasoning (mng.bz\/Nwr7).","logo":{"url":"https:\/\/yt3.ggpht.com\/WDMhk87M1UCdfKbYeOwXaYeUg5_8d7kuMOPKVI8f2schrd1L8M8Si3POxWtbZEZSqlDjX6BQxg=s800-c-k-c0x00ffffff-no-rj","width":800,"height":800,"@type":"ImageObject","@id":"https:\/\/www.youtube.com\/watch?v=quh7z1q7-uc#VideoObject_publisher_logo_ImageObject"}},"potentialAction":{"@type":"SeekToAction","@id":"https:\/\/www.youtube.com\/watch?v=quh7z1q7-uc#VideoObject_potentialAction","target":"https:\/\/www.youtube.com\/watch?v=quh7z1q7-uc&t={seek_to_second_number}","startOffset-input":"required name=seek_to_second_number"},"interactionStatistic":[[{"@type":"InteractionCounter","@id":"https:\/\/www.youtube.com\/watch?v=quh7z1q7-uc#VideoObject_interactionStatistic_WatchAction","interactionType":{"@type":"WatchAction"},"userInteractionCount":169666}],{"@type":"InteractionCounter","@id":"https:\/\/www.youtube.com\/watch?v=quh7z1q7-uc#VideoObject_interactionStatistic_LikeAction","interactionType":{"@type":"LikeAction"},"userInteractionCount":5169}]},"about":["Insights","\u300eEnglish\u300f"],"wordCount":1616},{"@context":"https:\/\/schema.org\/","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Insights","item":"https:\/\/blog.terabox.com\/insights\/#breadcrumbitem"},{"@type":"ListItem","position":2,"name":"Build Large Language Models (LLMs) from Scratch: A Guide","item":"https:\/\/blog.terabox.com\/insights\/build-large-language-models-from-scratch#breadcrumbitem"}]}]