[{"@context":"https:\/\/schema.org\/","@type":"BlogPosting","@id":"https:\/\/blog.terabox.com\/insights\/mamba-linear-time-sequence-modeling-guide#BlogPosting","mainEntityOfPage":"https:\/\/blog.terabox.com\/insights\/mamba-linear-time-sequence-modeling-guide","headline":"Mamba Architecture: Beyond the Transformer Bottleneck","name":"Mamba Architecture: Beyond the Transformer Bottleneck","description":"\ud83d\udcfa Today’s recommended deep-dive video: https:\/\/www.youtube.com\/watch?v=8Q_tqwpTpVU Mamba: The Linear-Time Architecture Challenging the Transformer\u2019s ThroneThe Evolution of Sequence ModelingFrom Bunny Populations to State Space ModelsThe Mamba Breakthrough: Selective ScanKey TakeawaysQ&A Mamba: The Linear-Time Architecture Challenging the Transformer\u2019s Throne For years, the... ","datePublished":"2026-08-18","dateModified":"2026-08-18","author":{"@type":"Person","@id":"https:\/\/blog.terabox.com\/author\/flextech-admin\/#Person","name":"flextech-admin","url":"https:\/\/blog.terabox.com\/author\/flextech-admin\/","image":{"@type":"ImageObject","@id":"https:\/\/secure.gravatar.com\/avatar\/ad516503a11cd5ca435acc9bb6523536?s=150&d=mm&r=gforcedefault=1","url":"https:\/\/secure.gravatar.com\/avatar\/ad516503a11cd5ca435acc9bb6523536?s=150&d=mm&r=gforcedefault=1","height":96,"width":96}},"publisher":{"@type":"Organization","name":"terabox","logo":{"@type":"ImageObject","@id":"http:\/\/blog.terabox.com\/wp-content\/uploads\/2021\/11\/logo\u4ea7\u54c1\u540d-\u7ad6\u7248.png","url":"http:\/\/blog.terabox.com\/wp-content\/uploads\/2021\/11\/logo\u4ea7\u54c1\u540d-\u7ad6\u7248.png","width":900,"height":900}},"image":{"@type":"ImageObject","@id":"https:\/\/img.youtube.com\/vi\/8Q_tqwpTpVU\/maxresdefault.jpg","url":"https:\/\/img.youtube.com\/vi\/8Q_tqwpTpVU\/maxresdefault.jpg","height":"","width":""},"url":"https:\/\/blog.terabox.com\/insights\/mamba-linear-time-sequence-modeling-guide","video":{"@context":"http:\/\/schema.org\/","@type":"VideoObject","@id":"https:\/\/www.youtube.com\/watch?v=8Q_tqwpTpVU#VideoObject","contentUrl":"https:\/\/www.youtube.com\/watch?v=8Q_tqwpTpVU","name":"Mamba and S4 Explained: Architecture, Parallel Scan, Kernel Fusion, Recurrent, Convolution, Math","description":"Explanation of the paper Mamba: Linear-Time Sequence Modeling with Selective State Spaces\n\nIn this video I will be explaining Mamba, a new sequence modeling architecture that can compete with the Transformer. I will first start by introducing the various sequence modeling architectures (RNN, CNN and Transformer) and then deep dive into State Space Models. To fully understand State Space Models, we need to have some background in differential equations. That's why, I will provide a brief introduction to differential equations (in 5 minutes!) and then proceed to derive the recurrent formula and the convolutional formula from first principles. I will also prove mathematically (with the help of visual diagrams) why State Space Models can be run as a convolution. I will explain what is the HIPPO matrix and how it can help the model \"memorize\" the input history in a finite state.\n\nIn the second part of the video, I will explore Mamba and in particular the Selective Scan algorithm, but first explaining what is the scan operation and how it can be parallelized, and then showing how the authors further improved the algorithm with Kernel Fusion and activations recomputation. I will also provide a brief lesson on the memory hierarchy in the GPU and why some operations may be IO-bound.\n\nIn the last part of the video we will explore the architecture of Mamba and some performance results to compare it with the Transformer.\n\nSlides PDF and Parallel Scan (excel file): https:\/\/github.com\/hkproj\/mamba-notes\n\nChapters\n00:00:00 - Introduction\n00:01:46 - Sequence modeling\n00:07:12 - Differential equations (basics)\n00:11:38 - State Space Models\n00:13:53 - Discretization\n00:23:08 - Recurrent computation\n00:26:32 - Convolutional computation\n00:34:18 - Skip connection term\n00:35:21 - Multidimentional SSM\n00:37:44 - The HIPPO theory\n00:43:30 - The motivation behind Mamba\n00:46:56 - Selective Scan algorithm\n00:51:34 - The Scan operation\n00:54:24 - Parallel Scan\n00:57:20 - Innovations in Selective Scan\n00:58:00 - GPU Memory Hierarchy\n01:01:23 - Kernel Fusion\n01:01:48 - Activations recomputation\n01:06:48 - Mamba architecture\n01:10:18 - Performance considerations\n01:12:54 - Conclusion","thumbnailUrl":["https:\/\/i.ytimg.com\/vi\/8Q_tqwpTpVU\/default.jpg","https:\/\/i.ytimg.com\/vi\/8Q_tqwpTpVU\/mqdefault.jpg","https:\/\/i.ytimg.com\/vi\/8Q_tqwpTpVU\/hqdefault.jpg","https:\/\/i.ytimg.com\/vi\/8Q_tqwpTpVU\/sddefault.jpg","https:\/\/i.ytimg.com\/vi\/8Q_tqwpTpVU\/maxresdefault.jpg"],"uploadDate":"2024-01-07T01:41:50+00:00","duration":"PT1H14M29S","embedUrl":"https:\/\/www.youtube.com\/embed\/8Q_tqwpTpVU","publisher":{"@type":"Organization","@id":"https:\/\/www.youtube.com\/channel\/UCtAcpQcYerN8xxZJYTfWBMw#Organization","url":"https:\/\/www.youtube.com\/channel\/UCtAcpQcYerN8xxZJYTfWBMw","name":"Umar Jamil","description":"I'm a Machine Learning Engineer from Milan, Italy, teaching complex deep learning and machine learning concepts to my cat, \u5965\u5229\u5965.\n\u6211\u4e5f\u4f1a\u4e2d\u6587.\n","logo":{"url":"https:\/\/yt3.ggpht.com\/prohEQLNtch3TRelWvLjr9aoUMBecNtLHqwO7WYlY0uM1yRP_69aa-gxINvsghoUyi2GvPJRv5w=s800-c-k-c0x00ffffff-no-rj","width":800,"height":800,"@type":"ImageObject","@id":"https:\/\/www.youtube.com\/watch?v=8Q_tqwpTpVU#VideoObject_publisher_logo_ImageObject"}},"potentialAction":{"@type":"SeekToAction","@id":"https:\/\/www.youtube.com\/watch?v=8Q_tqwpTpVU#VideoObject_potentialAction","target":"https:\/\/www.youtube.com\/watch?v=8Q_tqwpTpVU&t={seek_to_second_number}","startOffset-input":"required name=seek_to_second_number"},"interactionStatistic":[[{"@type":"InteractionCounter","@id":"https:\/\/www.youtube.com\/watch?v=8Q_tqwpTpVU#VideoObject_interactionStatistic_WatchAction","interactionType":{"@type":"WatchAction"},"userInteractionCount":64538}],{"@type":"InteractionCounter","@id":"https:\/\/www.youtube.com\/watch?v=8Q_tqwpTpVU#VideoObject_interactionStatistic_LikeAction","interactionType":{"@type":"LikeAction"},"userInteractionCount":2298}]},"about":["\u300eEnglish\u300f","Insights"],"wordCount":1501},{"@context":"https:\/\/schema.org\/","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Insights","item":"https:\/\/blog.terabox.com\/insights\/#breadcrumbitem"},{"@type":"ListItem","position":2,"name":"Mamba Architecture: Beyond the Transformer Bottleneck","item":"https:\/\/blog.terabox.com\/insights\/mamba-linear-time-sequence-modeling-guide#breadcrumbitem"}]}]