<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@regression_notes" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Notes – Chapter 7: Regression</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@html+block@regression_notes_top">
<div class="xblock xblock-public_view xblock-public_view-html xmodule_display xmodule_HtmlBlock" data-block-type="html" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@html+block@regression_notes_top" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "HTMLModule"}
</script>
<p>
You can sequence through the Regression lecture video and note segments (go to Next page). </p><p>
You can also (or alternatively) download the <a href="/assets/courseware/v1/030eba9b066b079e2b16dc863c18ea39/asset-v1:MITx+6.036+1T2019+type@asset+block/notes_chapter_Regression.pdf" target="_blank">Chapter 7: Regression</a> notes as a PDF file. </p>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@MIT6036L04a_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Lecture: Regression and the ordinary least squares problem</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04a">
<div class="xblock xblock-public_view xblock-public_view-video xmodule_display xmodule_VideoBlock" data-block-type="video" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04a" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "Video"}
</script>
<h3 class="hd hd-2">Lecture: Regression and the ordinary least squares problem</h3>
<div
id="video_MIT6036L04a"
class="video closed"
data-metadata='{"saveStateEnabled": false, "autoplay": false, "publishCompletionUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04a/handler/publish_completion", "start": 0.0, "prioritizeHls": false, "saveStateUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04a/handler/xmodule_handler/save_user_state", "recordedYoutubeIsAvailable": true, "streams": "1.00:GYY_VzamnaY", "autoAdvance": false, "transcriptAvailableTranslationsUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04a/handler/transcript/available_translations", "captionDataDir": null, "ytMetadataEndpoint": "", "showCaptions": "true", "lmsRootURL": "https://openlearninglibrary.mit.edu", "transcriptTranslationUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04a/handler/transcript/translation/__lang__", "ytApiUrl": "https://www.youtube.com/iframe_api", "transcriptLanguages": {"en": "English"}, "speed": null, "autohideHtml5": false, "generalSpeed": 1.0, "transcriptLanguage": "en", "savedVideoPosition": 0.0, "poster": null, "sources": [], "duration": 0.0, "end": 0.0, "completionEnabled": false, "completionPercentage": 0.95, "ytTestTimeout": 1500}'
data-bumper-metadata='null'
data-autoadvance-enabled="False"
data-poster='null'
tabindex="-1"
>
<div class="focus_grabber first"></div>
<div class="tc-wrapper">
<div class="video-wrapper">
<span tabindex="0" class="spinner" aria-hidden="false" aria-label="Loading video player"></span>
<span tabindex="-1" class="btn-play fa fa-youtube-play fa-2x is-hidden" aria-hidden="true" aria-label="Play video"></span>
<div class="video-player-pre"></div>
<div class="video-player">
<div id="MIT6036L04a"></div>
<h4 class="hd hd-4 video-error is-hidden">No playable video sources found.</h4>
<h4 class="hd hd-4 video-hls-error is-hidden">
Your browser does not support this video format. Try using a different browser.
</h4>
</div>
<div class="video-player-post"></div>
<div class="closed-captions"></div>
<div class="video-controls is-hidden">
<div>
<div class="vcr"><div class="vidtime">0:00 / 0:00</div></div>
<div class="secondary-controls"></div>
</div>
</div>
</div>
</div>
<div class="focus_grabber last"></div>
</div>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@regression_top_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Introduction to regression</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@html+block@regression_top">
<div class="xblock xblock-public_view xblock-public_view-html xmodule_display xmodule_HtmlBlock" data-block-type="html" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@html+block@regression_top" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "HTMLModule"}
</script>
<p>
Now we will turn to a slightly different form of machine-learning problem, called <span options="" class="marginote"><span class="marginote_desc" style="display:none">“Regression," in common parlance, means moving backwards. But this is forward progress!</span><span><em>regression</em></span></span> . It is still supervised learning, so our data will still have the form </p><table id="a0000000002" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]S_ n = \left\{ \left(x^{(1)}, y^{(1)}\right), \dots , \left(x^{(n)}, y^{(n)}\right)\right\} \; \; .[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
But now, instead of the [mathjaxinline]y[/mathjaxinline] values being discrete, they will be real-valued, and so our hypotheses will have the form </p><table id="a0000000003" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]h: \mathbb {R}^ d \rightarrow \mathbb {R} \; \; .[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
This is a good framework when we want to predict a numerical quantity, like height, stock value, etc., rather than to divide the inputs into categories. </p><p>
The first step is to pick a loss function, to describe how to evaluate the quality of the predictions our hypothesis is making, when compared to the “target" [mathjaxinline]y[/mathjaxinline] values in the data set. The choice of loss function is part of modeling your domain. In the absence of additional information about a regression problem, we typically use <i class="it">squared error</i> (SE): </p><table id="a0000000004" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]\text {Loss}(\text {guess}, \text {actual}) = (\text {guess} - \text {actual})^2 \; \; .[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
It penalizes guesses that are too high the same amount as it penalizes guesses that are too low, and has a good mathematical justification in the case that your data are generated from an underlying linear hypothesis, but with Gaussian-distributed noise added to the [mathjaxinline]y[/mathjaxinline] values. </p><p>
We will consider the case of a linear hypothesis class, </p><table id="a0000000005" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]h(x;\theta , \theta _0) = \theta ^ Tx + \theta _0 \; \; ,[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
remembering that we can get a rich class of hypotheses by performing a non-linear feature transformation before doing the regression. So, [mathjaxinline]\theta ^ Tx + \theta _0[/mathjaxinline] is a linear function of [mathjaxinline]x[/mathjaxinline], but [mathjaxinline]\theta ^ T\varphi (x) + \theta _0[/mathjaxinline] is a non-linear function of [mathjaxinline]x[/mathjaxinline] if [mathjaxinline]\varphi[/mathjaxinline] is a non-linear function of [mathjaxinline]x[/mathjaxinline]. </p><p>
We will treat regression as an optimization problem, in which, given a data set [mathjaxinline]{\cal D}[/mathjaxinline], we wish to find a linear hypothesis that minimizes mean squared error. Our objective, often called <em>mean squared error</em>, is to find values for [mathjaxinline]\Theta = (\theta , \theta _0)[/mathjaxinline] that minimize </p><table id="a0000000006" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]J(\theta , \theta _0) = \frac{1}{n}\sum _{i = 1}^ n\left(\theta ^ Tx^{(i)} + \theta _0 - y^{(i)}\right)^2 \; \; ,[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
resulting in the solution: </p><table id="a0000000007" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]\theta ^*, \theta _0^* = {\rm arg}\min _{\theta , \theta _0} J(\theta , \theta _0)\; \; .[/mathjax]</td><td class="eqnnum" style="width:20%; border:none;text-align:right">(1.1)</td></tr></table><p>
<br/></p><p>
<br/></p><p><a href="/assets/courseware/v1/030eba9b066b079e2b16dc863c18ea39/asset-v1:MITx+6.036+1T2019+type@asset+block/notes_chapter_Regression.pdf" target="_blank">Download this chapter as a PDF file</a></p><script src="/assets/courseware/v1/1ab2c06aefab58693cfc9c10394b7503/asset-v1:MITx+6.036+1T2019+type@asset+block/marginotes.js" type="text/javascript"/><span><br/><span style="color:gray;font-size:10pt"><center>This page was last updated on Saturday November 16, 2019; 07:31:13 PM (revision f808f068e)</center></span></span>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@MIT6036L04b_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Lecture: Regression - ordinary least squares solution using optimization</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04b">
<div class="xblock xblock-public_view xblock-public_view-video xmodule_display xmodule_VideoBlock" data-block-type="video" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04b" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "Video"}
</script>
<h3 class="hd hd-2">Lecture: Regression - ordinary least squares solution using optimization</h3>
<div
id="video_MIT6036L04b"
class="video closed"
data-metadata='{"saveStateEnabled": false, "autoplay": false, "publishCompletionUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04b/handler/publish_completion", "start": 0.0, "prioritizeHls": false, "saveStateUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04b/handler/xmodule_handler/save_user_state", "recordedYoutubeIsAvailable": true, "streams": "1.00:ZdZnwnXHllM", "autoAdvance": false, "transcriptAvailableTranslationsUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04b/handler/transcript/available_translations", "captionDataDir": null, "ytMetadataEndpoint": "", "showCaptions": "true", "lmsRootURL": "https://openlearninglibrary.mit.edu", "transcriptTranslationUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04b/handler/transcript/translation/__lang__", "ytApiUrl": "https://www.youtube.com/iframe_api", "transcriptLanguages": {"en": "English"}, "speed": null, "autohideHtml5": false, "generalSpeed": 1.0, "transcriptLanguage": "en", "savedVideoPosition": 0.0, "poster": null, "sources": [], "duration": 0.0, "end": 0.0, "completionEnabled": false, "completionPercentage": 0.95, "ytTestTimeout": 1500}'
data-bumper-metadata='null'
data-autoadvance-enabled="False"
data-poster='null'
tabindex="-1"
>
<div class="focus_grabber first"></div>
<div class="tc-wrapper">
<div class="video-wrapper">
<span tabindex="0" class="spinner" aria-hidden="false" aria-label="Loading video player"></span>
<span tabindex="-1" class="btn-play fa fa-youtube-play fa-2x is-hidden" aria-hidden="true" aria-label="Play video"></span>
<div class="video-player-pre"></div>
<div class="video-player">
<div id="MIT6036L04b"></div>
<h4 class="hd hd-4 video-error is-hidden">No playable video sources found.</h4>
<h4 class="hd hd-4 video-hls-error is-hidden">
Your browser does not support this video format. Try using a different browser.
</h4>
</div>
<div class="video-player-post"></div>
<div class="closed-captions"></div>
<div class="video-controls is-hidden">
<div>
<div class="vcr"><div class="vidtime">0:00 / 0:00</div></div>
<div class="secondary-controls"></div>
</div>
</div>
</div>
</div>
<div class="focus_grabber last"></div>
</div>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@MIT6036L04c_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Lecture: Regression - OLS analytical solution setup</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04c">
<div class="xblock xblock-public_view xblock-public_view-video xmodule_display xmodule_VideoBlock" data-block-type="video" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04c" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "Video"}
</script>
<h3 class="hd hd-2">Lecture: Regression - OLS analytical solution setup</h3>
<div
id="video_MIT6036L04c"
class="video closed"
data-metadata='{"saveStateEnabled": false, "autoplay": false, "publishCompletionUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04c/handler/publish_completion", "start": 0.0, "prioritizeHls": false, "saveStateUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04c/handler/xmodule_handler/save_user_state", "recordedYoutubeIsAvailable": true, "streams": "1.00:bmCSFd4d_Cc", "autoAdvance": false, "transcriptAvailableTranslationsUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04c/handler/transcript/available_translations", "captionDataDir": null, "ytMetadataEndpoint": "", "showCaptions": "true", "lmsRootURL": "https://openlearninglibrary.mit.edu", "transcriptTranslationUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04c/handler/transcript/translation/__lang__", "ytApiUrl": "https://www.youtube.com/iframe_api", "transcriptLanguages": {"en": "English"}, "speed": null, "autohideHtml5": false, "generalSpeed": 1.0, "transcriptLanguage": "en", "savedVideoPosition": 0.0, "poster": null, "sources": [], "duration": 0.0, "end": 0.0, "completionEnabled": false, "completionPercentage": 0.95, "ytTestTimeout": 1500}'
data-bumper-metadata='null'
data-autoadvance-enabled="False"
data-poster='null'
tabindex="-1"
>
<div class="focus_grabber first"></div>
<div class="tc-wrapper">
<div class="video-wrapper">
<span tabindex="0" class="spinner" aria-hidden="false" aria-label="Loading video player"></span>
<span tabindex="-1" class="btn-play fa fa-youtube-play fa-2x is-hidden" aria-hidden="true" aria-label="Play video"></span>
<div class="video-player-pre"></div>
<div class="video-player">
<div id="MIT6036L04c"></div>
<h4 class="hd hd-4 video-error is-hidden">No playable video sources found.</h4>
<h4 class="hd hd-4 video-hls-error is-hidden">
Your browser does not support this video format. Try using a different browser.
</h4>
</div>
<div class="video-player-post"></div>
<div class="closed-captions"></div>
<div class="video-controls is-hidden">
<div>
<div class="vcr"><div class="vidtime">0:00 / 0:00</div></div>
<div class="secondary-controls"></div>
</div>
</div>
</div>
</div>
<div class="focus_grabber last"></div>
</div>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@MIT6036L04d_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Lecture: Regression - OLS analytical solution using gradients</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04d">
<div class="xblock xblock-public_view xblock-public_view-video xmodule_display xmodule_VideoBlock" data-block-type="video" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04d" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "Video"}
</script>
<h3 class="hd hd-2">Lecture: Regression - OLS analytical solution using gradients</h3>
<div
id="video_MIT6036L04d"
class="video closed"
data-metadata='{"saveStateEnabled": false, "autoplay": false, "publishCompletionUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04d/handler/publish_completion", "start": 0.0, "prioritizeHls": false, "saveStateUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04d/handler/xmodule_handler/save_user_state", "recordedYoutubeIsAvailable": true, "streams": "1.00:sg9jErBC_pA", "autoAdvance": false, "transcriptAvailableTranslationsUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04d/handler/transcript/available_translations", "captionDataDir": null, "ytMetadataEndpoint": "", "showCaptions": "true", "lmsRootURL": "https://openlearninglibrary.mit.edu", "transcriptTranslationUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04d/handler/transcript/translation/__lang__", "ytApiUrl": "https://www.youtube.com/iframe_api", "transcriptLanguages": {"en": "English"}, "speed": null, "autohideHtml5": false, "generalSpeed": 1.0, "transcriptLanguage": "en", "savedVideoPosition": 0.0, "poster": null, "sources": [], "duration": 0.0, "end": 0.0, "completionEnabled": false, "completionPercentage": 0.95, "ytTestTimeout": 1500}'
data-bumper-metadata='null'
data-autoadvance-enabled="False"
data-poster='null'
tabindex="-1"
>
<div class="focus_grabber first"></div>
<div class="tc-wrapper">
<div class="video-wrapper">
<span tabindex="0" class="spinner" aria-hidden="false" aria-label="Loading video player"></span>
<span tabindex="-1" class="btn-play fa fa-youtube-play fa-2x is-hidden" aria-hidden="true" aria-label="Play video"></span>
<div class="video-player-pre"></div>
<div class="video-player">
<div id="MIT6036L04d"></div>
<h4 class="hd hd-4 video-error is-hidden">No playable video sources found.</h4>
<h4 class="hd hd-4 video-hls-error is-hidden">
Your browser does not support this video format. Try using a different browser.
</h4>
</div>
<div class="video-player-post"></div>
<div class="closed-captions"></div>
<div class="video-controls is-hidden">
<div>
<div class="vcr"><div class="vidtime">0:00 / 0:00</div></div>
<div class="secondary-controls"></div>
</div>
</div>
</div>
</div>
<div class="focus_grabber last"></div>
</div>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@regression_analytical_solution_ordinary_least_squares_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Analytical solution: ordinary least squares</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@html+block@regression_analytical_solution_ordinary_least_squares">
<div class="xblock xblock-public_view xblock-public_view-html xmodule_display xmodule_HtmlBlock" data-block-type="html" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@html+block@regression_analytical_solution_ordinary_least_squares" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "HTMLModule"}
</script>
<p>
One very interesting aspect of the problem finding a linear hypothesis that minimizes mean squared error (this general problem is often called <em>ordinary least squares</em> (<i class="sc">ols</i>)) is that we can find a closed-form formula <span options="" class="marginote"><span class="marginote_desc" style="display:none">What does “closed form" mean? Generally, that it involves direct evaluation of a mathematical expression using a fixed number of “typical" operations (like arithmetic operations, trig functions, powers, etc.). So equation <ref/> is not in closed form, because it's not at all clear what operations one needs to perform to find the solution.</span><span>for the answer!</span></span> </p><p>
Everything is easier to deal with if we assume that the [mathjaxinline]x^{(i)}[/mathjaxinline] have been augmented with an extra input dimension (feature) that always has value 1, so we may ignore [mathjaxinline]\theta _0[/mathjaxinline]. (See chapter <ref/>, section <ref/> for a reminder about this strategy). </p><p>
We will approach this just like a minimization problem from calculus homework: take the derivative of [mathjaxinline]J[/mathjaxinline] with respect to [mathjaxinline]\theta[/mathjaxinline], set it to zero, and solve for [mathjaxinline]\theta[/mathjaxinline]. There is an additional step required, to check that the resulting [mathjaxinline]\theta[/mathjaxinline] is a minimum (rather than a maximum or an inflection point) but we won't work through that here. It is possible to approach this problem by: </p><ul class="itemize"><li><p>
Finding [mathjaxinline]\partial {J}/\partial {\theta _ k}[/mathjaxinline] for [mathjaxinline]k[/mathjaxinline] in <span options="" class="marginote"><span class="marginote_desc" style="display:none">We will use [mathjaxinline]d[/mathjaxinline] here for the total number of features in each [mathjaxinline]x^{(i)}[/mathjaxinline], including the added 1.</span><span>[mathjaxinline]1, \ldots , d[/mathjaxinline],</span></span> </p></li><li><p>
Constructing a set of [mathjaxinline]k[/mathjaxinline] equations of the form [mathjaxinline]\partial {J}/\partial {\theta _ k} = 0[/mathjaxinline], and </p></li><li><p>
Solving the system for values of [mathjaxinline]\theta _ k[/mathjaxinline]. </p></li></ul><p>
That works just fine. To get practice for applying techniques like this to more complex problems, we will work through a more compact (and cool!) matrix view. <br/> <br/><span style="color:#FF0000"><b class="bf">Study Question:</b></span> <span style="color:#0000FF">Work through this and check your answer against ours below.</span> <br/></p><p>
We can think of our training data in terms of matrices [mathjaxinline]X[/mathjaxinline] and [mathjaxinline]Y[/mathjaxinline], where each column of [mathjaxinline]X[/mathjaxinline] is an example, and each “column" of [mathjaxinline]Y[/mathjaxinline] is the corresponding target output value: </p><table id="a0000000008" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]X = \begin{bmatrix} x_1^{(1)} & \dots & x_1^{(n)}\\ \vdots & \ddots & \vdots \\ x_ d^{(1)} & \dots & x_ d^{(n)}\end{bmatrix} \; \; \; Y = \begin{bmatrix} y^{(1)} & \dots & y^{(n)}\end{bmatrix}\; \; .[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
<br/> <br/><span style="color:#FF0000"><b class="bf">Study Question:</b></span> <span style="color:#0000FF"> What are the dimensions of [mathjaxinline]X[/mathjaxinline] and [mathjaxinline]Y[/mathjaxinline]? </span> <br/></p><p>
In most textbooks, they think of an individual example [mathjaxinline]x^{(i)}[/mathjaxinline] as a row, rather than a column. So that we get an answer that will be recognizable to you, we are going to define a new matrix and vector, [mathjaxinline]W[/mathjaxinline] and [mathjaxinline]T[/mathjaxinline], which are just transposes of our [mathjaxinline]X[/mathjaxinline] and [mathjaxinline]Y[/mathjaxinline], and then work with them: </p><table id="a0000000009" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]W = X^ T = \begin{bmatrix} x_1^{(1)} & \dots & x_ d^{(1)}\\ \vdots & \ddots & \vdots \\ x_1^{(n)} & \dots & x_ d^{(n)}\end{bmatrix} \; \; T = Y^ T = \begin{bmatrix} y^{(1)}\\ \vdots \\ y^{(n)}\end{bmatrix} \; \; .[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
<br/> <br/><span style="color:#FF0000"><b class="bf">Study Question:</b></span> <span style="color:#0000FF"> What are the dimensions of [mathjaxinline]W[/mathjaxinline] and [mathjaxinline]T[/mathjaxinline]? </span> <br/></p><p>
Now we can write </p><table id="a0000000010" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]J(\theta ) = \frac{1}{n}\underbrace{(W\theta - T)^ T}_{1 \times n}\underbrace{(W\theta - T)}_{n \times 1} = \frac{1}{n}\sum _{i=1}^ n \left(\left(\sum _{j=1}^ d W_{ij}\theta _ j \right) - T_ i\right)^2[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
and using facts about matrix/vector calculus, we get </p><table id="a0000000011" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]\nabla _{\theta }J = \frac{2}{n}\underbrace{W^ T}_{d \times n}\underbrace{(W\theta - T)}_{n \times 1}\; \; .[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
Setting to 0 and solving, we get: </p><table id="a0000000012" cellpadding="7" width="100%" cellspacing="0" class="eqnarray" style="table-layout:auto"><tr id="a0000000013"><td style="width:40%; border:none"> </td><td style="vertical-align:middle; text-align:right; border:none">
[mathjaxinline]\displaystyle \frac{2}{n}W^ T(W\theta - T)[/mathjaxinline]
</td><td style="vertical-align:middle; text-align:left; border:none">
[mathjaxinline]\displaystyle = 0[/mathjaxinline]
</td><td style="width:40%; border:none"> </td><td style="width:20%; border:none" class="eqnnum"> </td></tr><tr id="a0000000014"><td style="width:40%; border:none"> </td><td style="vertical-align:middle; text-align:right; border:none">
[mathjaxinline]\displaystyle W^ TW\theta - W^ T T[/mathjaxinline]
</td><td style="vertical-align:middle; text-align:left; border:none">
[mathjaxinline]\displaystyle = 0[/mathjaxinline]
</td><td style="width:40%; border:none"> </td><td style="width:20%; border:none" class="eqnnum"> </td></tr><tr id="a0000000015"><td style="width:40%; border:none"> </td><td style="vertical-align:middle; text-align:right; border:none">
[mathjaxinline]\displaystyle W^ TW\theta[/mathjaxinline]
</td><td style="vertical-align:middle; text-align:left; border:none">
[mathjaxinline]\displaystyle = W^ T T[/mathjaxinline]
</td><td style="width:40%; border:none"> </td><td style="width:20%; border:none" class="eqnnum"> </td></tr><tr id="a0000000016"><td style="width:40%; border:none"> </td><td style="vertical-align:middle; text-align:right; border:none">
[mathjaxinline]\displaystyle \theta[/mathjaxinline]
</td><td style="vertical-align:middle; text-align:left; border:none">
[mathjaxinline]\displaystyle = (W^ TW)^{-1} W^ T T[/mathjaxinline]
</td><td style="width:40%; border:none"> </td><td style="width:20%; border:none" class="eqnnum"> </td></tr></table><p>
And the dimensions work out! </p><table id="a0000000017" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]\theta = \underbrace{\left(W^ TW\right)^{-1}}_{d \times d}\underbrace{W^ T}_{d \times n}\underbrace{T}_{n \times 1}[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
So, given our data, we can directly compute the linear regression that minimizes mean squared error. That's pretty awesome! </p><p>
<br/></p><p>
<br/></p><p><a href="/assets/courseware/v1/030eba9b066b079e2b16dc863c18ea39/asset-v1:MITx+6.036+1T2019+type@asset+block/notes_chapter_Regression.pdf" target="_blank">Download this chapter as a PDF file</a></p><script src="/assets/courseware/v1/1ab2c06aefab58693cfc9c10394b7503/asset-v1:MITx+6.036+1T2019+type@asset+block/marginotes.js" type="text/javascript"/><span><br/><span style="color:gray;font-size:10pt"><center>This page was last updated on Friday May 24, 2019; 02:28:45 PM (revision 4f166135)</center></span></span>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@MIT6036L04e_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Lecture: Regression - beauty of the closed form OLS solution</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04e">
<div class="xblock xblock-public_view xblock-public_view-video xmodule_display xmodule_VideoBlock" data-block-type="video" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04e" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "Video"}
</script>
<h3 class="hd hd-2">Lecture: Regression - beauty of the closed form OLS solution</h3>
<div
id="video_MIT6036L04e"
class="video closed"
data-metadata='{"saveStateEnabled": false, "autoplay": false, "publishCompletionUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04e/handler/publish_completion", "start": 0.0, "prioritizeHls": false, "saveStateUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04e/handler/xmodule_handler/save_user_state", "recordedYoutubeIsAvailable": true, "streams": "1.00:zZkH5u6f1Hw", "autoAdvance": false, "transcriptAvailableTranslationsUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04e/handler/transcript/available_translations", "captionDataDir": null, "ytMetadataEndpoint": "", "showCaptions": "true", "lmsRootURL": "https://openlearninglibrary.mit.edu", "transcriptTranslationUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04e/handler/transcript/translation/__lang__", "ytApiUrl": "https://www.youtube.com/iframe_api", "transcriptLanguages": {"en": "English"}, "speed": null, "autohideHtml5": false, "generalSpeed": 1.0, "transcriptLanguage": "en", "savedVideoPosition": 0.0, "poster": null, "sources": [], "duration": 0.0, "end": 0.0, "completionEnabled": false, "completionPercentage": 0.95, "ytTestTimeout": 1500}'
data-bumper-metadata='null'
data-autoadvance-enabled="False"
data-poster='null'
tabindex="-1"
>
<div class="focus_grabber first"></div>
<div class="tc-wrapper">
<div class="video-wrapper">
<span tabindex="0" class="spinner" aria-hidden="false" aria-label="Loading video player"></span>
<span tabindex="-1" class="btn-play fa fa-youtube-play fa-2x is-hidden" aria-hidden="true" aria-label="Play video"></span>
<div class="video-player-pre"></div>
<div class="video-player">
<div id="MIT6036L04e"></div>
<h4 class="hd hd-4 video-error is-hidden">No playable video sources found.</h4>
<h4 class="hd hd-4 video-hls-error is-hidden">
Your browser does not support this video format. Try using a different browser.
</h4>
</div>
<div class="video-player-post"></div>
<div class="closed-captions"></div>
<div class="video-controls is-hidden">
<div>
<div class="vcr"><div class="vidtime">0:00 / 0:00</div></div>
<div class="secondary-controls"></div>
</div>
</div>
</div>
</div>
<div class="focus_grabber last"></div>
</div>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@MIT6036L04f_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Lecture: Regression - regularization by ridge regression</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04f">
<div class="xblock xblock-public_view xblock-public_view-video xmodule_display xmodule_VideoBlock" data-block-type="video" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04f" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "Video"}
</script>
<h3 class="hd hd-2">Lecture: Regression - regularization by ridge regression</h3>
<div
id="video_MIT6036L04f"
class="video closed"
data-metadata='{"saveStateEnabled": false, "autoplay": false, "publishCompletionUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04f/handler/publish_completion", "start": 0.0, "prioritizeHls": false, "saveStateUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04f/handler/xmodule_handler/save_user_state", "recordedYoutubeIsAvailable": true, "streams": "1.00:DpVEKoBcQHY", "autoAdvance": false, "transcriptAvailableTranslationsUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04f/handler/transcript/available_translations", "captionDataDir": null, "ytMetadataEndpoint": "", "showCaptions": "true", "lmsRootURL": "https://openlearninglibrary.mit.edu", "transcriptTranslationUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04f/handler/transcript/translation/__lang__", "ytApiUrl": "https://www.youtube.com/iframe_api", "transcriptLanguages": {"en": "English"}, "speed": null, "autohideHtml5": false, "generalSpeed": 1.0, "transcriptLanguage": "en", "savedVideoPosition": 0.0, "poster": null, "sources": [], "duration": 0.0, "end": 0.0, "completionEnabled": false, "completionPercentage": 0.95, "ytTestTimeout": 1500}'
data-bumper-metadata='null'
data-autoadvance-enabled="False"
data-poster='null'
tabindex="-1"
>
<div class="focus_grabber first"></div>
<div class="tc-wrapper">
<div class="video-wrapper">
<span tabindex="0" class="spinner" aria-hidden="false" aria-label="Loading video player"></span>
<span tabindex="-1" class="btn-play fa fa-youtube-play fa-2x is-hidden" aria-hidden="true" aria-label="Play video"></span>
<div class="video-player-pre"></div>
<div class="video-player">
<div id="MIT6036L04f"></div>
<h4 class="hd hd-4 video-error is-hidden">No playable video sources found.</h4>
<h4 class="hd hd-4 video-hls-error is-hidden">
Your browser does not support this video format. Try using a different browser.
</h4>
</div>
<div class="video-player-post"></div>
<div class="closed-captions"></div>
<div class="video-controls is-hidden">
<div>
<div class="vcr"><div class="vidtime">0:00 / 0:00</div></div>
<div class="secondary-controls"></div>
</div>
</div>
</div>
</div>
<div class="focus_grabber last"></div>
</div>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@MIT6036L04g_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Lecture: Regression - analytical minimization of the ridge regression objective</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04g">
<div class="xblock xblock-public_view xblock-public_view-video xmodule_display xmodule_VideoBlock" data-block-type="video" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04g" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "Video"}
</script>
<h3 class="hd hd-2">Lecture: Regression - analytical minimization of the ridge regression objective</h3>
<div
id="video_MIT6036L04g"
class="video closed"
data-metadata='{"saveStateEnabled": false, "autoplay": false, "publishCompletionUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04g/handler/publish_completion", "start": 0.0, "prioritizeHls": false, "saveStateUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04g/handler/xmodule_handler/save_user_state", "recordedYoutubeIsAvailable": true, "streams": "1.00:aUYKpLZGvu4", "autoAdvance": false, "transcriptAvailableTranslationsUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04g/handler/transcript/available_translations", "captionDataDir": null, "ytMetadataEndpoint": "", "showCaptions": "true", "lmsRootURL": "https://openlearninglibrary.mit.edu", "transcriptTranslationUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04g/handler/transcript/translation/__lang__", "ytApiUrl": "https://www.youtube.com/iframe_api", "transcriptLanguages": {"en": "English"}, "speed": null, "autohideHtml5": false, "generalSpeed": 1.0, "transcriptLanguage": "en", "savedVideoPosition": 0.0, "poster": null, "sources": [], "duration": 0.0, "end": 0.0, "completionEnabled": false, "completionPercentage": 0.95, "ytTestTimeout": 1500}'
data-bumper-metadata='null'
data-autoadvance-enabled="False"
data-poster='null'
tabindex="-1"
>
<div class="focus_grabber first"></div>
<div class="tc-wrapper">
<div class="video-wrapper">
<span tabindex="0" class="spinner" aria-hidden="false" aria-label="Loading video player"></span>
<span tabindex="-1" class="btn-play fa fa-youtube-play fa-2x is-hidden" aria-hidden="true" aria-label="Play video"></span>
<div class="video-player-pre"></div>
<div class="video-player">
<div id="MIT6036L04g"></div>
<h4 class="hd hd-4 video-error is-hidden">No playable video sources found.</h4>
<h4 class="hd hd-4 video-hls-error is-hidden">
Your browser does not support this video format. Try using a different browser.
</h4>
</div>
<div class="video-player-post"></div>
<div class="closed-captions"></div>
<div class="video-controls is-hidden">
<div>
<div class="vcr"><div class="vidtime">0:00 / 0:00</div></div>
<div class="secondary-controls"></div>
</div>
</div>
</div>
</div>
<div class="focus_grabber last"></div>
</div>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@regression_regularizing_linear_regression_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Regularizing linear regression</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@html+block@regression_regularizing_linear_regression">
<div class="xblock xblock-public_view xblock-public_view-html xmodule_display xmodule_HtmlBlock" data-block-type="html" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@html+block@regression_regularizing_linear_regression" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "HTMLModule"}
</script>
<p>
Well, actually, there are some kinds of trouble we can get into. What if [mathjaxinline]\left(W^ TW\right)[/mathjaxinline] is not invertible? <br/> <br/><span style="color:#FF0000"><b class="bf">Study Question:</b></span> <span style="color:#0000FF">Consider, for example, a situation where the data-set is just the same point repeated twice: [mathjaxinline]x^{(1)} = x^{(2)} = (1, 2)^ T[/mathjaxinline]. What is [mathjaxinline]W[/mathjaxinline] in this case? What is [mathjaxinline]W^ TW[/mathjaxinline]? What is [mathjaxinline](W^ TW)^{-1}[/mathjaxinline]? </span> <br/></p><p>
Another kind of problem is <em>overfitting</em>: we have formulated an objective that is just about fitting the data as well as possible, but as we discussed in the context of margin maximization, we might also want to <em>regularize</em> to keep the hypothesis from getting <em>too</em> attached to the data. </p><p>
We address both the problem of not being able to invert [mathjaxinline](W^ TW)^{-1}[/mathjaxinline] and the problem of overfitting using a mechanism called <em>ridge regression</em>. We add a regularization term [mathjaxinline]\| \theta \| ^2[/mathjaxinline] to the <i class="sc">ols</i> objective, with trade-off parameter [mathjaxinline]\lambda[/mathjaxinline]. <br/> <br/><span style="color:#FF0000"><b class="bf">Study Question:</b></span> <span style="color:#0000FF">When we add a regularizer of the form [mathjaxinline]\| \theta \| ^2[/mathjaxinline], what is our most “preferred" value of [mathjaxinline]\theta[/mathjaxinline], in the absence of any data?</span> <br/>Here is the ridge regression objective function: </p><table id="a0000000018" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]J_{\text {ridge}}(\theta , \theta _0) = \frac{1}{n}\sum _{i = 1}^ n\left(\theta ^ Tx^{(i)} + \theta _0 - y^{(i)}\right)^2 + \lambda \| \theta \| ^2[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
Larger [mathjaxinline]\lambda[/mathjaxinline] values pressure [mathjaxinline]\theta[/mathjaxinline] values to be near zero. Note that we don't penalize [mathjaxinline]\theta _0[/mathjaxinline]; intuitively, [mathjaxinline]\theta _0[/mathjaxinline] is what “floats" the regression surface to the right level for the data you have, and so you shouldn't make it harder to fit a data set where the [mathjaxinline]y[/mathjaxinline] values tend to be around one million than one where they tend to be around one. The other parameters control the orientation of the regression surface, and we prefer it to have a not-too-crazy orientation. </p><p>
There is an analytical expression for the [mathjaxinline]\theta , \theta _0[/mathjaxinline] values that minimize [mathjaxinline]J_\text {ridge}[/mathjaxinline], but it's a little bit more complicated to derive than the solution for <i class="sc">ols</i> because [mathjaxinline]\theta _0[/mathjaxinline] needs special treatment. If we decide not to treat [mathjaxinline]\theta _0[/mathjaxinline] specially (so we add a 1 feature to our input vectors), then we get: </p><table id="a0000000019" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]\nabla _{\theta }J_\text {ridge} = \frac{2}{n}W^ T(W\theta - T) + 2 \lambda \theta \; \; .[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
Setting to 0 and solving, we get: </p><table id="a0000000020" cellpadding="7" width="100%" cellspacing="0" class="eqnarray" style="table-layout:auto"><tr id="a0000000021"><td style="width:40%; border:none"> </td><td style="vertical-align:middle; text-align:right; border:none">
[mathjaxinline]\displaystyle \frac{2}{n}W^ T(W\theta - T) + 2 \lambda \theta[/mathjaxinline]
</td><td style="vertical-align:middle; text-align:left; border:none">
[mathjaxinline]\displaystyle = 0[/mathjaxinline]
</td><td style="width:40%; border:none"> </td><td style="width:20%; border:none" class="eqnnum"> </td></tr><tr id="a0000000022"><td style="width:40%; border:none"> </td><td style="vertical-align:middle; text-align:right; border:none">
[mathjaxinline]\displaystyle \frac{1}{n}W^ TW\theta - \frac{1}{n}W^ TT + \lambda \theta[/mathjaxinline]
</td><td style="vertical-align:middle; text-align:left; border:none">
[mathjaxinline]\displaystyle = 0[/mathjaxinline]
</td><td style="width:40%; border:none"> </td><td style="width:20%; border:none" class="eqnnum"> </td></tr><tr id="a0000000023"><td style="width:40%; border:none"> </td><td style="vertical-align:middle; text-align:right; border:none">
[mathjaxinline]\displaystyle \frac{1}{n}W^ TW\theta + \lambda \theta[/mathjaxinline]
</td><td style="vertical-align:middle; text-align:left; border:none">
[mathjaxinline]\displaystyle = \frac{1}{n}W^ TT[/mathjaxinline]
</td><td style="width:40%; border:none"> </td><td style="width:20%; border:none" class="eqnnum"> </td></tr><tr id="a0000000024"><td style="width:40%; border:none"> </td><td style="vertical-align:middle; text-align:right; border:none">
[mathjaxinline]\displaystyle W^ TW\theta + n \lambda \theta[/mathjaxinline]
</td><td style="vertical-align:middle; text-align:left; border:none">
[mathjaxinline]\displaystyle = W^ TT[/mathjaxinline]
</td><td style="width:40%; border:none"> </td><td style="width:20%; border:none" class="eqnnum"> </td></tr><tr id="a0000000025"><td style="width:40%; border:none"> </td><td style="vertical-align:middle; text-align:right; border:none">
[mathjaxinline]\displaystyle (W^ TW + n \lambda I)\theta[/mathjaxinline]
</td><td style="vertical-align:middle; text-align:left; border:none">
[mathjaxinline]\displaystyle = W^ TT[/mathjaxinline]
</td><td style="width:40%; border:none"> </td><td style="width:20%; border:none" class="eqnnum"> </td></tr><tr id="a0000000026"><td style="width:40%; border:none"> </td><td style="vertical-align:middle; text-align:right; border:none">
[mathjaxinline]\displaystyle \theta[/mathjaxinline]
</td><td style="vertical-align:middle; text-align:left; border:none">
[mathjaxinline]\displaystyle = (W^ TW + n \lambda I)^{-1}W^ TT[/mathjaxinline]
</td><td style="width:40%; border:none"> </td><td style="width:20%; border:none" class="eqnnum"> </td></tr></table><p>
Whew! So, </p><table id="a0000000027" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]\theta _{\text {ridge}} = \left(W^ TW + n\lambda I\right)^{-1}W^ TT[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p><span options="" class="marginote"><span class="marginote_desc" style="display:none">This is called “ridge" regression because we are adding a “ridge" of [mathjaxinline]\lambda[/mathjaxinline] values along the diagonal of the matrix before inverting it.</span><span>which </span></span> becomes invertible when [mathjaxinline]\lambda > 0[/mathjaxinline]. <br/> <br/><span style="color:#FF0000"><b class="bf">Study Question:</b></span> <span style="color:#0000FF">Derive this version of the ridge regression solution.</span> <br/></p><p><span style="color:ForestGreen"><b class="bf">Talking about regularization</b></span></p><p>
In machine learning in general, not just regression, it is useful to distinguish two ways in which a hypothesis [mathjaxinline]h \in {\cal H}[/mathjaxinline] might contribute to errors on test data. We have </p><p><img src="/assets/courseware/v1/86806188792b74e101c37dabab525641/asset-v1:MITx+6.036+1T2019+type@asset+block/images_regression_regularizing_linear_regression_description_1-crop.png" width="895"/></p><p><span options="" class="marginote"><span class="marginote_desc" style="display:none">There are technical definitions of these concepts that are studied in more advanced treatments of machine learning. Structural error is referred to as <em>bias</em> and estimation error is referred to as <em>variance</em>.</span><span>note</span></span> When we increase [mathjaxinline]\lambda[/mathjaxinline], we tend to increase structural error but decrease estimation error, and vice versa. </p><p>
<br/> <br/><span style="color:#FF0000"><b class="bf">Study Question:</b></span> <span style="color:#0000FF"> Consider using a polynomial basis of order [mathjaxinline]k[/mathjaxinline] as a feature transformation [mathjaxinline]\phi[/mathjaxinline] on your data. Would increasing [mathjaxinline]k[/mathjaxinline] tend to increase or decrease structural error? What about estimation error? </span> <br/></p><p>
<br/></p><p>
<br/></p><p><a href="/assets/courseware/v1/030eba9b066b079e2b16dc863c18ea39/asset-v1:MITx+6.036+1T2019+type@asset+block/notes_chapter_Regression.pdf" target="_blank">Download this chapter as a PDF file</a></p><script src="/assets/courseware/v1/1ab2c06aefab58693cfc9c10394b7503/asset-v1:MITx+6.036+1T2019+type@asset+block/marginotes.js" type="text/javascript"/><span><br/><span style="color:gray;font-size:10pt"><center>This page was last updated on Saturday November 16, 2019; 07:31:13 PM (revision f808f068e)</center></span></span>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@MIT6036L04h_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Lecture: Regression - ridge regression using gradient descent</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04h">
<div class="xblock xblock-public_view xblock-public_view-video xmodule_display xmodule_VideoBlock" data-block-type="video" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04h" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "Video"}
</script>
<h3 class="hd hd-2">Lecture: Regression - ridge regression using gradient descent</h3>
<div
id="video_MIT6036L04h"
class="video closed"
data-metadata='{"saveStateEnabled": false, "autoplay": false, "publishCompletionUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04h/handler/publish_completion", "start": 0.0, "prioritizeHls": false, "saveStateUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04h/handler/xmodule_handler/save_user_state", "recordedYoutubeIsAvailable": true, "streams": "1.00:e2m444vvVf0", "autoAdvance": false, "transcriptAvailableTranslationsUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04h/handler/transcript/available_translations", "captionDataDir": null, "ytMetadataEndpoint": "", "showCaptions": "true", "lmsRootURL": "https://openlearninglibrary.mit.edu", "transcriptTranslationUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04h/handler/transcript/translation/__lang__", "ytApiUrl": "https://www.youtube.com/iframe_api", "transcriptLanguages": {"en": "English"}, "speed": null, "autohideHtml5": false, "generalSpeed": 1.0, "transcriptLanguage": "en", "savedVideoPosition": 0.0, "poster": null, "sources": [], "duration": 0.0, "end": 0.0, "completionEnabled": false, "completionPercentage": 0.95, "ytTestTimeout": 1500}'
data-bumper-metadata='null'
data-autoadvance-enabled="False"
data-poster='null'
tabindex="-1"
>
<div class="focus_grabber first"></div>
<div class="tc-wrapper">
<div class="video-wrapper">
<span tabindex="0" class="spinner" aria-hidden="false" aria-label="Loading video player"></span>
<span tabindex="-1" class="btn-play fa fa-youtube-play fa-2x is-hidden" aria-hidden="true" aria-label="Play video"></span>
<div class="video-player-pre"></div>
<div class="video-player">
<div id="MIT6036L04h"></div>
<h4 class="hd hd-4 video-error is-hidden">No playable video sources found.</h4>
<h4 class="hd hd-4 video-hls-error is-hidden">
Your browser does not support this video format. Try using a different browser.
</h4>
</div>
<div class="video-player-post"></div>
<div class="closed-captions"></div>
<div class="video-controls is-hidden">
<div>
<div class="vcr"><div class="vidtime">0:00 / 0:00</div></div>
<div class="secondary-controls"></div>
</div>
</div>
</div>
</div>
<div class="focus_grabber last"></div>
</div>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@regression_optimization_via_gradient_descent_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Optimization via gradient descent</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@html+block@regression_optimization_via_gradient_descent">
<div class="xblock xblock-public_view xblock-public_view-html xmodule_display xmodule_HtmlBlock" data-block-type="html" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@html+block@regression_optimization_via_gradient_descent" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "HTMLModule"}
</script>
<p>
Inverting the [mathjaxinline]d \times d[/mathjaxinline] matrix [mathjaxinline]\left(W^ TW\right)[/mathjaxinline] takes <span options="" class="marginote"><span class="marginote_desc" style="display:none">Well, actually, Gauss-Jordan elimination, a popular algorithm, takes [mathjaxinline]O(d^3)[/mathjaxinline] arithmetic operations, but the bit complexity of the intermediate results can grow exponentially! There are other algorithms with polynomial bit complexity. (If this just made no sense to you, don't worry.)</span><span>[mathjaxinline]O(d^3)[/mathjaxinline]</span></span> time, which makes the analytic solution impractical for large [mathjaxinline]d[/mathjaxinline]. If we have high-dimensional data, we can fall back on gradient descent. <br/> <br/><span style="color:#FF0000"><b class="bf">Study Question:</b></span> <span style="color:#0000FF">Why is having large [mathjaxinline]n[/mathjaxinline] not as much of a computational problem as having large [mathjaxinline]d[/mathjaxinline]?</span> <br/></p><p>
Recall the ridge objective </p><table id="a0000000028" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]J_{\text {ridge}}(\theta , \theta _0) = \frac{1}{n}\sum _{i = 1}^ n\left(\theta ^ Tx^{(i)} + \theta _0 - y^{(i)}\right)^2 + \lambda \| \theta \| ^2[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
and its gradient with respect to [mathjaxinline]\theta[/mathjaxinline] </p><table id="a0000000029" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]\nabla _{\theta }J = \frac{2}{n}\sum _{i = 1}^ n\left(\theta ^ Tx^{(i)} + \theta _0 - y^{(i)}\right)x^{(i)} + 2\lambda \theta[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
and partial derivative with respect to [mathjaxinline]\theta _0[/mathjaxinline] </p><table id="a0000000030" class="equation" width="100%" cellspacing="0" cellpadding="7" style="table-layout:auto"><tr><td class="equation" style="width:80%; border:none">[mathjax]\frac{\partial J}{\partial \theta _0} = \frac{2}{n}\sum _{i = 1}^ n\left(\theta ^ Tx^{(i)} + \theta _0 - y^{(i)}\right) \; \; .[/mathjax]</td><td class="eqnnum" style="width:20%; border:none"> </td></tr></table><p>
Armed with these derivatives, we can do gradient descent, using the regular or stochastic gradient methods from chapter <ref/>. </p><p>
Even better, the objective functions for <i class="sc">ols</i> and ridge regression are <em>convex</em>, which means they have only one minimum, which means, with a small enough step size, gradient descent is <em>guaranteed</em> to find the optimum. </p><p>
<br/></p><p>
<br/></p><p><a href="/assets/courseware/v1/030eba9b066b079e2b16dc863c18ea39/asset-v1:MITx+6.036+1T2019+type@asset+block/notes_chapter_Regression.pdf" target="_blank">Download this chapter as a PDF file</a></p><script src="/assets/courseware/v1/1ab2c06aefab58693cfc9c10394b7503/asset-v1:MITx+6.036+1T2019+type@asset+block/marginotes.js" type="text/javascript"/><span><br/><span style="color:gray;font-size:10pt"><center>This page was last updated on Friday May 24, 2019; 02:28:45 PM (revision 4f166135)</center></span></span>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@MIT6036L04j_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Lecture: Regression - stochastic gradient descent</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04j">
<div class="xblock xblock-public_view xblock-public_view-video xmodule_display xmodule_VideoBlock" data-block-type="video" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04j" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "Video"}
</script>
<h3 class="hd hd-2">Lecture: Regression - stochastic gradient descent</h3>
<div
id="video_MIT6036L04j"
class="video closed"
data-metadata='{"saveStateEnabled": false, "autoplay": false, "publishCompletionUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04j/handler/publish_completion", "start": 0.0, "prioritizeHls": false, "saveStateUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04j/handler/xmodule_handler/save_user_state", "recordedYoutubeIsAvailable": true, "streams": "1.00:O7M7xyUcxTU", "autoAdvance": false, "transcriptAvailableTranslationsUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04j/handler/transcript/available_translations", "captionDataDir": null, "ytMetadataEndpoint": "", "showCaptions": "true", "lmsRootURL": "https://openlearninglibrary.mit.edu", "transcriptTranslationUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04j/handler/transcript/translation/__lang__", "ytApiUrl": "https://www.youtube.com/iframe_api", "transcriptLanguages": {"en": "English"}, "speed": null, "autohideHtml5": false, "generalSpeed": 1.0, "transcriptLanguage": "en", "savedVideoPosition": 0.0, "poster": null, "sources": [], "duration": 0.0, "end": 0.0, "completionEnabled": false, "completionPercentage": 0.95, "ytTestTimeout": 1500}'
data-bumper-metadata='null'
data-autoadvance-enabled="False"
data-poster='null'
tabindex="-1"
>
<div class="focus_grabber first"></div>
<div class="tc-wrapper">
<div class="video-wrapper">
<span tabindex="0" class="spinner" aria-hidden="false" aria-label="Loading video player"></span>
<span tabindex="-1" class="btn-play fa fa-youtube-play fa-2x is-hidden" aria-hidden="true" aria-label="Play video"></span>
<div class="video-player-pre"></div>
<div class="video-player">
<div id="MIT6036L04j"></div>
<h4 class="hd hd-4 video-error is-hidden">No playable video sources found.</h4>
<h4 class="hd hd-4 video-hls-error is-hidden">
Your browser does not support this video format. Try using a different browser.
</h4>
</div>
<div class="video-player-post"></div>
<div class="closed-captions"></div>
<div class="video-controls is-hidden">
<div>
<div class="vcr"><div class="vidtime">0:00 / 0:00</div></div>
<div class="secondary-controls"></div>
</div>
</div>
</div>
</div>
<div class="focus_grabber last"></div>
</div>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@MIT6036L04k_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Lecture: Regression - OLS and gradient descent demo example</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04k">
<div class="xblock xblock-public_view xblock-public_view-video xmodule_display xmodule_VideoBlock" data-block-type="video" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04k" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "Video"}
</script>
<h3 class="hd hd-2">Lecture: Regression - OLS and gradient descent demo example</h3>
<div
id="video_MIT6036L04k"
class="video closed"
data-metadata='{"saveStateEnabled": false, "autoplay": false, "publishCompletionUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04k/handler/publish_completion", "start": 0.0, "prioritizeHls": false, "saveStateUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04k/handler/xmodule_handler/save_user_state", "recordedYoutubeIsAvailable": true, "streams": "1.00:-72TMOHAQnc", "autoAdvance": false, "transcriptAvailableTranslationsUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04k/handler/transcript/available_translations", "captionDataDir": null, "ytMetadataEndpoint": "", "showCaptions": "true", "lmsRootURL": "https://openlearninglibrary.mit.edu", "transcriptTranslationUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04k/handler/transcript/translation/__lang__", "ytApiUrl": "https://www.youtube.com/iframe_api", "transcriptLanguages": {"en": "English"}, "speed": null, "autohideHtml5": false, "generalSpeed": 1.0, "transcriptLanguage": "en", "savedVideoPosition": 0.0, "poster": null, "sources": [], "duration": 0.0, "end": 0.0, "completionEnabled": false, "completionPercentage": 0.95, "ytTestTimeout": 1500}'
data-bumper-metadata='null'
data-autoadvance-enabled="False"
data-poster='null'
tabindex="-1"
>
<div class="focus_grabber first"></div>
<div class="tc-wrapper">
<div class="video-wrapper">
<span tabindex="0" class="spinner" aria-hidden="false" aria-label="Loading video player"></span>
<span tabindex="-1" class="btn-play fa fa-youtube-play fa-2x is-hidden" aria-hidden="true" aria-label="Play video"></span>
<div class="video-player-pre"></div>
<div class="video-player">
<div id="MIT6036L04k"></div>
<h4 class="hd hd-4 video-error is-hidden">No playable video sources found.</h4>
<h4 class="hd hd-4 video-hls-error is-hidden">
Your browser does not support this video format. Try using a different browser.
</h4>
</div>
<div class="video-player-post"></div>
<div class="closed-captions"></div>
<div class="video-controls is-hidden">
<div>
<div class="vcr"><div class="vidtime">0:00 / 0:00</div></div>
<div class="secondary-controls"></div>
</div>
</div>
</div>
</div>
<div class="focus_grabber last"></div>
</div>
</div>
</div>
</div>
</div>
<div class="xblock xblock-public_view xblock-public_view-vertical" data-block-type="vertical" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="VerticalStudentView" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@vertical+block@MIT6036L04m_vert" data-graded="False" data-runtime-class="LmsRuntime">
<h2 class="hd hd-2 unit-title">Lecture: Regression - structural error and estimation error</h2>
<div class="vert-mod">
<div class="vert vert-0" data-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04m">
<div class="xblock xblock-public_view xblock-public_view-video xmodule_display xmodule_VideoBlock" data-block-type="video" data-has-score="False" data-runtime-version="1" data-course-id="course-v1:MITx+6.036+1T2019" data-init="XBlockToXModuleShim" data-request-token="5332866e03e511f099b402fa081815af" data-usage-id="block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04m" data-graded="False" data-runtime-class="LmsRuntime">
<script type="json/xblock-args" class="xblock-json-init-args">
{"xmodule-type": "Video"}
</script>
<h3 class="hd hd-2">Lecture: Regression - structural error and estimation error</h3>
<div
id="video_MIT6036L04m"
class="video closed"
data-metadata='{"saveStateEnabled": false, "autoplay": false, "publishCompletionUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04m/handler/publish_completion", "start": 0.0, "prioritizeHls": false, "saveStateUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04m/handler/xmodule_handler/save_user_state", "recordedYoutubeIsAvailable": true, "streams": "1.00:7-G8ZFL2oiE", "autoAdvance": false, "transcriptAvailableTranslationsUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04m/handler/transcript/available_translations", "captionDataDir": null, "ytMetadataEndpoint": "", "showCaptions": "true", "lmsRootURL": "https://openlearninglibrary.mit.edu", "transcriptTranslationUrl": "/courses/course-v1:MITx+6.036+1T2019/xblock/block-v1:MITx+6.036+1T2019+type@video+block@MIT6036L04m/handler/transcript/translation/__lang__", "ytApiUrl": "https://www.youtube.com/iframe_api", "transcriptLanguages": {"en": "English"}, "speed": null, "autohideHtml5": false, "generalSpeed": 1.0, "transcriptLanguage": "en", "savedVideoPosition": 0.0, "poster": null, "sources": [], "duration": 0.0, "end": 0.0, "completionEnabled": false, "completionPercentage": 0.95, "ytTestTimeout": 1500}'
data-bumper-metadata='null'
data-autoadvance-enabled="False"
data-poster='null'
tabindex="-1"
>
<div class="focus_grabber first"></div>
<div class="tc-wrapper">
<div class="video-wrapper">
<span tabindex="0" class="spinner" aria-hidden="false" aria-label="Loading video player"></span>
<span tabindex="-1" class="btn-play fa fa-youtube-play fa-2x is-hidden" aria-hidden="true" aria-label="Play video"></span>
<div class="video-player-pre"></div>
<div class="video-player">
<div id="MIT6036L04m"></div>
<h4 class="hd hd-4 video-error is-hidden">No playable video sources found.</h4>
<h4 class="hd hd-4 video-hls-error is-hidden">
Your browser does not support this video format. Try using a different browser.
</h4>
</div>
<div class="video-player-post"></div>
<div class="closed-captions"></div>
<div class="video-controls is-hidden">
<div>
<div class="vcr"><div class="vidtime">0:00 / 0:00</div></div>
<div class="secondary-controls"></div>
</div>
</div>
</div>
</div>
<div class="focus_grabber last"></div>
</div>
</div>
</div>
</div>
</div>
© All Rights Reserved