angular.module('docsIsolationExample', [])
  .controller('Ctrl', function($scope, $log) {
    $scope.naomi = { name: 'Naomi', address: '1600 Amphitheatre' };

    $scope.vojta = { name: 'Vojta', address: '3456 Somewhere Else' };
    
    $scope.facList = [{"Id":7214,"Name":"VALUE1"},{"Id":1003,"Name":"VALUE2"},{"Id":7217,"Name":"VALUE3"},{"Id":7216,"Name":"VALUE4"},{"Id":7215,"Name":"VALUE5"},{"Id":7213,"Name":"VALUE6"},{"Id":6310,"Name":"VALUE7"},{"Id":7200,"Name":"VALUE8"},{"Id":1007,"Name":"VALUE9"},{"Id":1000,"Name":"VALUE10"},{"Id":7211,"Name":"VALUE11"},{"Id":1004,"Name":"VALUE12"},{"Id":2597,"Name":"VALUE13"},{"Id":2042,"Name":"VALUE14"},{"Id":7218,"Name":"VALUE15"},{"Id":7222,"Name":"VALUE16"}];
    $scope.selected = [];
    
    
    $scope.AddItem = function () {
        var items = angular.element("#sortFac li.selectedSortListItem:not('.ui-state-disabled')");
        
        for (var k = 0; k < items.length; k++) {
            var scopeItem = angular.element(items[k]).scope();
            
            //-- Find current element position every time as we are removing elements from the array
            var currIdx = $scope.facList.indexOf(scopeItem.obj);

            if (currIdx > -1) {
                $scope.facList.splice(currIdx, 1);
                $scope.selected.push(scopeItem.obj);
            }            
        }        
    }


    $scope.RemoveItem = function () {
        var items = angular.element("#sortFac2 li.selectedSortListItem:not('.ui-state-disabled')");

        for (var k = 0; k < items.length; k++) {
            var scopeItem = angular.element(items[k]).scope();

            //-- Find current element position every time as we are removing elements from the array
            var currIdx = $scope.selected.indexOf(scopeItem.obj);

            if (currIdx > -1) {
                $scope.selected.splice(currIdx, 1);
                $scope.facList.push(scopeItem.obj);
            }
        }
    }
  })
  .directive('sortableList', function ($log, $parse) {
    return {
        restrict: 'A',
       // scope: true,
        scope: {
            fromList: '=',
            toList: '='
        },
        link: function (scope, elm, attrs) {                        

            var callback = {
                receive: function (event, ui) {
                    $log.log(scope);
                    $log.log(attrs);
                    //-- Get the scope of the list-item
                    var scopeItem = ui.item.scope();
                    //-- Get new list index
                    var newIdx = ui.item.index();
                    
                    //-- Find position in the list
                    var prevIdx = scope.fromList.indexOf(scopeItem.obj);                    

                    //-- Remove from source list
                    scope.fromList.splice(prevIdx, 1);
                    //-- Add to target list
                    if (newIdx >= scope.toList.length) {
                      scope.toList.push(scopeItem.obj);
                        //scope.$parent[attrs.toList].splice(newIdx, scope.$parent[attrs.toList].length-1, scopeItem.obj);
                    }
                    else {
                      $log.log(newIdx);
                      //$log.log(scope.$parent[attrs.toList]);
                        scope.toList.splice(newIdx, 0, scopeItem.obj);
                    }

                    ui.item.removeClass('selectedSortListItem').addClass('sortListItem');

                    //scope.$apply();
                },
                stop: function (event, ui) {
                    //$log.log(ui);
                }
            };            
            
            //-- Apply jquery ui sortable plug-in to element
            elm.sortable({
                handle: ".handle",
                connectWith: '.sortColumnsConnect',
                dropOnEmpty: true,
                cancel: ".ui-state-disabled",
                receive: callback.receive,
                stop: callback.stop
            }).disableSelection();

            
            
        }
    }
})
.directive('sortableListElm', function () {
    return {
        restrict: 'A',
        link: function (scope, elm) {
            elm.click(function () {
                $(this).toggleClass('selectedSortListItem').toggleClass('sortListItem');
            });
                    
        }
    }
});
<!doctype html>
<html ng-app="docsIsolationExample">
  <head>
  <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
<script type="text/javascript" src="http://code.angularjs.org/1.2.13/angular.min.js"></script>

<link rel="stylesheet" href="//ajax.googleapis.com/ajax/libs/jqueryui/1.10.4/themes/smoothness/jquery-ui.css" />
<script src="//ajax.googleapis.com/ajax/libs/jqueryui/1.10.4/jquery-ui.min.js"></script>

<script type="text/javascript" src="script.js"></script>
<link href="style.css" rel="stylesheet" type="text/css" /> 
</head>
  <body>

<pre>
  Sortable list should work in two ways. 
  - Selecting elements from the list, then click the action button.
  - Drag n' Drop an element from either list to the other.
  
  Add button adds a selected element from the lst list to the right list.
  Remove button removes an element from the right list ot the left one.
  
  Problem: Select one or more elements from the left list and click add. 
  Then drag-n-drop an element from the left list to the right list. Now, select a new element
  on the left and click add. 
  If you keep adding to the left using the action button now the directive seems to be broken even the
  results are correctly added to the list.
  Now, try to remove the added elements from the right list. (Select one or more, then click remove).
  If you try to remove them all in multiple steps, you might notice that at least one can't be 
  selected nor removed. Now try to add one element from the right to the left using the action button.
  EXPLOTION!!!
  
  Help is appreciated!!!
</pre>


<div ng-controller="Ctrl">
  
  <div class="box">
      <ul sortable-list from-list="selected" to-list="facList" id="sortFac" class="sortColumnsConnect" style="padding-bottom:100px; padding-left: 0px;">

          <li class="sortListItem ui-state-disabled">SELECT ALL</li>
          
           <li sortable-list-elm class="sortListItem" ng-repeat="obj in facList">
              <span class="handle" ></span>{{ obj.Id + ' - ' + obj.Name }}
           </li>
         
      </ul>
  </div>
  
  <div style="display:inline-block;">
    <button type="button" ng-click="AddItem()" class="green icon-btn">Add<span class="icon-add"></span></button>  
    <br/>
    <button class="red icon-btn" style="margin: 6px 0px;" type="button" ng-click="RemoveItem()"><span class="icon-remove"></span>Remove</button>  
  </div>
  

  <div class="box">
      <ul sortable-list to-list="selected" from-list="facList" id="sortFac2" class="sortColumnsConnect" style="padding-bottom:100px; padding-left: 0px;">
          
          <li sortable-list-elm class="sortListItem" ng-repeat="obj in selected">
              <span class="handle"></span>{{ obj.Id + ' - ' + obj.Name }}
          </li>
                     
      </ul>
  </div>
  
  <br/>
  
  {{ selected }}
  
</div>


  </body>
</html>
.box {
 overflow: auto; height:275px; width:300px;display:inline-block;border: 1px solid black;
}

li .handle {
position: absolute;
left: 0px;
top: 0px;
width: 20px;
height: 22px;
background-image: url(http://thinkflood.com/support/redeye/wp-content/uploads/2011/05/handle-icon-actions.png);
background-repeat: no-repeat;
background-position: center;
}

.sortListItem
{
    position: relative;
    margin: 3px;
    width: 150px;
    background-color: #f8f8f8;
    background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #f8f8f8), color-stop(100%, #e1dede));
    background-image: -webkit-linear-gradient(top, #f8f8f8, #e1dede);
    background-image: -moz-linear-gradient(top, #f8f8f8, #e1dede);
    background-image: -ms-linear-gradient(top, #f8f8f8, #e1dede);
    background-image: -o-linear-gradient(top, #f8f8f8, #e1dede);
    background-image: linear-gradient(top, #f8f8f8, #e1dede);
    border: 1px solid #aaaaaa;
    border-bottom: 1px solid #777676;
    -webkit-border-radius: 3px;
    -moz-border-radius: 3px;
    border-radius: 3px;
    -webkit-box-shadow: inset 0 1px 0 0 #fff;
    -moz-box-shadow: inset 0 1px 0 0 #fff;
    box-shadow: inset 0 1px 0 0 #fff;
    color: #4a4a4a;
    line-height: 1;
    padding: 6px 10px 6px 21px;
    text-shadow: 0 -1px 0 #fff;
    font-weight:bold;
    font-size:10px;
}

.selectedSortListItem
{
    position: relative;
    margin: 3px;
    width: 160px;
    background-color: #99CCFF;
    background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #f8f8f8), color-stop(100%, #99CCFF));
    background-image: -webkit-linear-gradient(top, #f8f8f8, #99CCFF);
    background-image: -moz-linear-gradient(top, #f8f8f8, #99CCFF);
    background-image: -ms-linear-gradient(top, #f8f8f8, #99CCFF);
    background-image: -o-linear-gradient(top, #f8f8f8, #99CCFF);
    background-image: linear-gradient(top, #f8f8f8, #99CCFF);
    border: 1px solid #aaaaaa;
    border-bottom: 1px solid #777676;
    -webkit-border-radius: 3px;
    -moz-border-radius: 3px;
    border-radius: 3px;
    -webkit-box-shadow: inset 0 1px 0 0 #fff;
    -moz-box-shadow: inset 0 1px 0 0 #fff;
    box-shadow: inset 0 1px 0 0 #fff;
    color: #4a4a4a;
    line-height: 1;
    padding: 6px 10px 6px 21px;
    text-shadow: 0 -1px 0 #fff;
    font-weight:bold;
    font-size:10px;
}